Coverage for python/lsst/daf/butler/_butler.py: 29%
303 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-06 01:33 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 **kwargs : `Any`
142 Additional keyword arguments passed to a constructor of actual butler
143 class.
145 Notes
146 -----
147 The preferred way to instantiate Butler is via the `from_config` method.
148 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
149 but ``mypy`` will complain about the former.
150 """
152 def __new__(
153 cls,
154 config: Config | ResourcePathExpression | None = None,
155 *,
156 collections: Any = None,
157 run: str | None = None,
158 searchPaths: Sequence[ResourcePathExpression] | None = None,
159 writeable: bool | None = None,
160 inferDefaults: bool = True,
161 without_datastore: bool = False,
162 metrics: ButlerMetrics | None = None,
163 **kwargs: Any,
164 ) -> Butler:
165 if cls is Butler:
166 return Butler.from_config(
167 config=config,
168 collections=collections,
169 run=run,
170 searchPaths=searchPaths,
171 writeable=writeable,
172 inferDefaults=inferDefaults,
173 without_datastore=without_datastore,
174 metrics=metrics,
175 **kwargs,
176 )
178 # Note: we do not pass any parameters to __new__, Python will pass them
179 # to __init__ after __new__ returns sub-class instance.
180 return super().__new__(cls)
182 @classmethod
183 def from_config(
184 cls,
185 config: Config | ResourcePathExpression | None = None,
186 *,
187 collections: Any = None,
188 run: str | None = None,
189 searchPaths: Sequence[ResourcePathExpression] | None = None,
190 writeable: bool | None = None,
191 inferDefaults: bool = True,
192 without_datastore: bool = False,
193 metrics: ButlerMetrics | None = None,
194 **kwargs: Any,
195 ) -> Butler:
196 """Create butler instance from configuration.
198 Parameters
199 ----------
200 config : `ButlerConfig`, `Config` or `str`, optional
201 Configuration. Anything acceptable to the `ButlerConfig`
202 constructor. If a directory path is given the configuration will be
203 read from a ``butler.yaml`` file in that location. If `None` is
204 given default values will be used. If ``config`` contains "cls" key
205 then its value is used as a name of butler class and it must be a
206 sub-class of this class, otherwise `DirectButler` is instantiated.
207 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
208 An expression specifying the collections to be searched (in order)
209 when reading datasets.
210 This may be a `str` collection name or an iterable thereof.
211 See :ref:`daf_butler_collection_expressions` for more information.
212 These collections are not registered automatically and must be
213 manually registered before they are used by any method, but they
214 may be manually registered after the `Butler` is initialized.
215 run : `str`, optional
216 Name of the `~CollectionType.RUN` collection new datasets should be
217 inserted into. If ``collections`` is `None` and ``run`` is not
218 `None`, ``collections`` will be set to ``[run]``. If not `None`,
219 this collection will automatically be registered. If this is not
220 set (and ``writeable`` is not set either), a read-only butler will
221 be created.
222 searchPaths : `list` of `str`, optional
223 Directory paths to search when calculating the full Butler
224 configuration. Not used if the supplied config is already a
225 `ButlerConfig`.
226 writeable : `bool`, optional
227 Explicitly sets whether the butler supports write operations. If
228 not provided, a read-write butler is created if any of ``run``,
229 ``tags``, or ``chains`` is non-empty.
230 inferDefaults : `bool`, optional
231 If `True` (default) infer default data ID values from the values
232 present in the datasets in ``collections``: if all collections have
233 the same value (or no value) for a governor dimension, that value
234 will be the default for that dimension. Nonexistent collections
235 are ignored. If a default value is provided explicitly for a
236 governor dimension via ``**kwargs``, no default will be inferred
237 for that dimension.
238 without_datastore : `bool`, optional
239 If `True` do not attach a datastore to this butler. Any attempts
240 to use a datastore will fail.
241 metrics : `ButlerMetrics` or `None`, optional
242 Metrics object to record butler usage statistics.
243 **kwargs : `Any`
244 Default data ID key-value pairs. These may only identify
245 "governor" dimensions like ``instrument`` and ``skymap``.
247 Returns
248 -------
249 butler : `Butler`
250 A `Butler` constructed from the given configuration.
252 Notes
253 -----
254 Calling this factory method is identical to calling
255 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
256 complains about ``Butler()`` call.
258 Examples
259 --------
260 While there are many ways to control exactly how a `Butler` interacts
261 with the collections in its `Registry`, the most common cases are still
262 simple.
264 For a read-only `Butler` that searches one collection, do::
266 butler = Butler.from_config(
267 "/path/to/repo", collections=["u/alice/DM-50000"]
268 )
270 For a read-write `Butler` that writes to and reads from a
271 `~CollectionType.RUN` collection::
273 butler = Butler.from_config(
274 "/path/to/repo", run="u/alice/DM-50000/a"
275 )
277 The `Butler` passed to a ``PipelineTask`` is often much more complex,
278 because we want to write to one `~CollectionType.RUN` collection but
279 read from several others (as well)::
281 butler = Butler.from_config(
282 "/path/to/repo",
283 run="u/alice/DM-50000/a",
284 collections=[
285 "u/alice/DM-50000/a",
286 "u/bob/DM-49998",
287 "HSC/defaults",
288 ],
289 )
291 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
292 Datasets will be read first from that run (since it appears first in
293 the chain), and then from ``u/bob/DM-49998`` and finally
294 ``HSC/defaults``.
296 Finally, one can always create a `Butler` with no collections::
298 butler = Butler.from_config("/path/to/repo", writeable=True)
300 This can be extremely useful when you just want to use
301 ``butler.registry``, e.g. for inserting dimension data or managing
302 collections, or when the collections you want to use with the butler
303 are not consistent. Passing ``writeable`` explicitly here is only
304 necessary if you want to be able to make changes to the repo - usually
305 the value for ``writeable`` can be guessed from the collection
306 arguments provided, but it defaults to `False` when there are not
307 collection arguments.
308 """
309 # DirectButler used to have a way to specify a "copy constructor" by
310 # passing the "butler" parameter to its constructor. This has
311 # been moved out of the constructor into Butler.clone().
312 butler = kwargs.pop("butler", None)
313 metrics = metrics if metrics is not None else ButlerMetrics()
314 if butler is not None:
315 if not isinstance(butler, Butler):
316 raise TypeError("'butler' parameter must be a Butler instance")
317 if config is not None or searchPaths is not None or writeable is not None:
318 raise TypeError(
319 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
320 )
321 return butler.clone(
322 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
323 )
325 options = ButlerInstanceOptions(
326 collections=collections,
327 run=run,
328 writeable=writeable,
329 inferDefaults=inferDefaults,
330 metrics=metrics,
331 kwargs=kwargs,
332 )
334 # Load the Butler configuration. This may involve searching the
335 # environment to locate a configuration file.
336 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
337 butler_type = butler_config.get_butler_type()
339 # Make DirectButler if class is not specified.
340 match butler_type:
341 case ButlerType.DIRECT:
342 from .direct_butler import DirectButler
344 return DirectButler.create_from_config(
345 butler_config,
346 options=options,
347 without_datastore=without_datastore,
348 )
349 case ButlerType.REMOTE:
350 from .remote_butler._factory import RemoteButlerFactory
352 # Assume this is being created by a client who would like
353 # default caching of remote datasets.
354 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
355 return factory.create_butler_with_credentials_from_environment(
356 butler_options=options, enable_datastore_cache=True
357 )
358 case _:
359 raise TypeError(f"Unknown Butler type '{butler_type}'")
361 @staticmethod
362 def makeRepo(
363 root: ResourcePathExpression,
364 config: Config | str | None = None,
365 dimensionConfig: Config | str | None = None,
366 standalone: bool = False,
367 searchPaths: list[str] | None = None,
368 forceConfigRoot: bool = True,
369 outfile: ResourcePathExpression | None = None,
370 overwrite: bool = False,
371 ) -> Config:
372 """Create an empty data repository by adding a butler.yaml config
373 to a repository root directory.
375 Parameters
376 ----------
377 root : `lsst.resources.ResourcePathExpression`
378 Path or URI to the root location of the new repository. Will be
379 created if it does not exist.
380 config : `Config` or `str`, optional
381 Configuration to write to the repository, after setting any
382 root-dependent Registry or Datastore config options. Can not
383 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
384 configuration will be used. Root-dependent config options
385 specified in this config are overwritten if ``forceConfigRoot``
386 is `True`.
387 dimensionConfig : `Config` or `str`, optional
388 Configuration for dimensions, will be used to initialize registry
389 database.
390 standalone : `bool`
391 If True, write all expanded defaults, not just customized or
392 repository-specific settings.
393 This (mostly) decouples the repository from the default
394 configuration, insulating it from changes to the defaults (which
395 may be good or bad, depending on the nature of the changes).
396 Future *additions* to the defaults will still be picked up when
397 initializing `Butlers` to repos created with ``standalone=True``.
398 searchPaths : `list` of `str`, optional
399 Directory paths to search when calculating the full butler
400 configuration.
401 forceConfigRoot : `bool`, optional
402 If `False`, any values present in the supplied ``config`` that
403 would normally be reset are not overridden and will appear
404 directly in the output config. This allows non-standard overrides
405 of the root directory for a datastore or registry to be given.
406 If this parameter is `True` the values for ``root`` will be
407 forced into the resulting config if appropriate.
408 outfile : `lss.resources.ResourcePathExpression`, optional
409 If not-`None`, the output configuration will be written to this
410 location rather than into the repository itself. Can be a URI
411 string. Can refer to a directory that will be used to write
412 ``butler.yaml``.
413 overwrite : `bool`, optional
414 Create a new configuration file even if one already exists
415 in the specified output location. Default is to raise
416 an exception.
418 Returns
419 -------
420 config : `Config`
421 The updated `Config` instance written to the repo.
423 Raises
424 ------
425 ValueError
426 Raised if a ButlerConfig or ConfigSubset is passed instead of a
427 regular Config (as these subclasses would make it impossible to
428 support ``standalone=False``).
429 FileExistsError
430 Raised if the output config file already exists.
431 os.error
432 Raised if the directory does not exist, exists but is not a
433 directory, or cannot be created.
435 Notes
436 -----
437 Note that when ``standalone=False`` (the default), the configuration
438 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
439 construct the repository should also be used to construct any Butlers
440 to avoid configuration inconsistencies.
441 """
442 if isinstance(config, ButlerConfig | ConfigSubset):
443 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
445 # Ensure that the root of the repository exists or can be made
446 root_uri = ResourcePath(root, forceDirectory=True)
447 root_uri.mkdir()
449 config = Config(config)
451 # If we are creating a new repo from scratch with relative roots,
452 # do not propagate an explicit root from the config file
453 if "root" in config:
454 del config["root"]
456 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
457 imported_class = doImportType(full["datastore", "cls"])
458 if not issubclass(imported_class, Datastore):
459 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
460 datastoreClass: type[Datastore] = imported_class
461 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
463 # if key exists in given config, parse it, otherwise parse the defaults
464 # in the expanded config
465 if config.get(("registry", "db")):
466 registryConfig = RegistryConfig(config)
467 else:
468 registryConfig = RegistryConfig(full)
469 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
470 if defaultDatabaseUri is not None:
471 Config.updateParameters(
472 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
473 )
474 else:
475 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
477 if standalone:
478 config.merge(full)
479 else:
480 # Always expand the registry.managers section into the per-repo
481 # config, because after the database schema is created, it's not
482 # allowed to change anymore. Note that in the standalone=True
483 # branch, _everything_ in the config is expanded, so there's no
484 # need to special case this.
485 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
486 configURI: ResourcePathExpression
487 if outfile is not None:
488 # When writing to a separate location we must include
489 # the root of the butler repo in the config else it won't know
490 # where to look.
491 config["root"] = root_uri.geturl()
492 configURI = outfile
493 else:
494 configURI = root_uri
495 # Strip obscore configuration, if it is present, before writing config
496 # to a file, obscore config will be stored in registry.
497 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
498 config_to_write = config.copy()
499 del config_to_write[obscore_config_key]
500 config_to_write.dumpToUri(configURI, overwrite=overwrite)
501 # configFile attribute is updated, need to copy it to original.
502 config.configFile = config_to_write.configFile
503 else:
504 config.dumpToUri(configURI, overwrite=overwrite)
506 # Create Registry and populate tables
507 registryConfig = RegistryConfig(config.get("registry"))
508 dimensionConfig = DimensionConfig(dimensionConfig)
509 _RegistryFactory(registryConfig).create_from_config(
510 dimensionConfig=dimensionConfig, butlerRoot=root_uri
511 )
513 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
515 return config
517 @classmethod
518 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
519 """Look up the label in a butler repository index.
521 Parameters
522 ----------
523 label : `str`
524 Label of the Butler repository to look up.
525 return_label : `bool`, optional
526 If ``label`` cannot be found in the repository index (either
527 because index is not defined or ``label`` is not in the index) and
528 ``return_label`` is `True` then return ``ResourcePath(label)``.
529 If ``return_label`` is `False` (default) then an exception will be
530 raised instead.
532 Returns
533 -------
534 uri : `lsst.resources.ResourcePath`
535 URI to the Butler repository associated with the given label or
536 default value if it is provided.
538 Raises
539 ------
540 KeyError
541 Raised if the label is not found in the index, or if an index
542 is not defined, and ``return_label`` is `False`.
544 Notes
545 -----
546 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
547 information is discovered.
548 """
549 return ButlerRepoIndex.get_repo_uri(label, return_label)
551 @classmethod
552 def get_known_repos(cls) -> set[str]:
553 """Retrieve the list of known repository labels.
555 Returns
556 -------
557 repos : `set` of `str`
558 All the known labels. Can be empty if no index can be found.
560 Notes
561 -----
562 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
563 information is discovered.
564 """
565 return ButlerRepoIndex.get_known_repos()
567 @classmethod
568 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
569 """Extract the butler label and dataset ID from a dataset URI.
571 Parameters
572 ----------
573 uri : `str`
574 The dataset URI to parse.
576 Returns
577 -------
578 parsed : `ParsedButlerDatasetURI`
579 The label associated with the butler repository from which this
580 dataset originates and the ID of the dataset.
582 Notes
583 -----
584 Supports dataset URIs of the forms
585 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
586 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
587 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
588 can include anything supported by the `Butler` constructor, including
589 paths to repositories and alias labels.
591 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
593 will return a label of ``/repo/main``.
595 This method does not attempt to check that the dataset exists in the
596 labeled butler.
598 Since the IVOID can be issued by any publisher to represent a Butler
599 dataset there is no validation of the path or netloc component of the
600 URI. The only requirement is that there are ``id`` and ``repo`` keys
601 in the ``ivo`` URI query component.
602 """
603 parsed = urllib.parse.urlparse(uri)
604 parsed_scheme = parsed.scheme.lower()
605 if parsed_scheme == "ivo":
606 # Do not validate the netloc or the path values.
607 qs = urllib.parse.parse_qs(parsed.query)
608 if "repo" not in qs or "id" not in qs:
609 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
610 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
611 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
612 label = qs["repo"][0]
613 id_ = qs["id"][0]
614 elif parsed_scheme == "butler":
615 label = parsed.netloc # Butler label is case sensitive.
616 # Need to strip the leading /.
617 id_ = parsed.path[1:]
618 else:
619 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
620 # Strip trailing/leading whitespace from label.
621 label = label.strip()
622 if not label:
623 raise ValueError(f"No butler repository label found in uri {uri!r}")
624 try:
625 dataset_id = uuid.UUID(hex=id_)
626 except Exception as e:
627 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
628 raise
630 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
632 @classmethod
633 def get_dataset_from_uri(
634 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
635 ) -> SpecificButlerDataset:
636 """Get the dataset associated with the given dataset URI.
638 Parameters
639 ----------
640 uri : `str`
641 The URI associated with a dataset.
642 factory : `LabeledButlerFactoryProtocol` or `None`, optional
643 Bound factory function that will be given the butler label
644 and receive a `Butler`. If this is not provided the label
645 will be tried directly.
647 Returns
648 -------
649 result : `SpecificButlerDataset`
650 The butler associated with this URI and the dataset itself.
651 The dataset can be `None` if the UUID is valid but the dataset
652 is not known to this butler.
653 """
654 parsed = cls.parse_dataset_uri(uri)
655 butler: Butler | None = None
656 if factory is not None:
657 # If the label is not recognized, it might be a path.
658 try:
659 butler = factory(parsed.label)
660 except KeyError:
661 pass
662 if butler is None:
663 butler = cls.from_config(parsed.label)
664 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
666 @abstractmethod
667 def _caching_context(self) -> AbstractContextManager[None]:
668 """Context manager that enables caching."""
669 raise NotImplementedError()
671 @abstractmethod
672 def transaction(self) -> AbstractContextManager[None]:
673 """Context manager supporting `Butler` transactions.
675 Transactions can be nested.
676 """
677 raise NotImplementedError()
679 @abstractmethod
680 def put(
681 self,
682 obj: Any,
683 datasetRefOrType: DatasetRef | DatasetType | str,
684 /,
685 dataId: DataId | None = None,
686 *,
687 run: str | None = None,
688 provenance: DatasetProvenance | None = None,
689 **kwargs: Any,
690 ) -> DatasetRef:
691 """Store and register a dataset.
693 Parameters
694 ----------
695 obj : `object`
696 The dataset.
697 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
698 When `DatasetRef` is provided, ``dataId`` should be `None`.
699 Otherwise the `DatasetType` or name thereof. If a fully resolved
700 `DatasetRef` is given the run and ID are used directly.
701 dataId : `dict` or `DataCoordinate`
702 A `dict` of `Dimension` link name, value pairs that label the
703 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
704 should be provided as the second argument.
705 run : `str`, optional
706 The name of the run the dataset should be added to, overriding
707 ``self.run``. Not used if a resolved `DatasetRef` is provided.
708 provenance : `DatasetProvenance` or `None`, optional
709 Any provenance that should be attached to the serialized dataset.
710 Not supported by all serialization mechanisms.
711 **kwargs
712 Additional keyword arguments used to augment or construct a
713 `DataCoordinate`. See `DataCoordinate.standardize`
714 parameters. Not used if a resolve `DatasetRef` is provided.
716 Returns
717 -------
718 ref : `DatasetRef`
719 A reference to the stored dataset, updated with the correct id if
720 given.
722 Raises
723 ------
724 TypeError
725 Raised if the butler is read-only or if no run has been provided.
726 """
727 raise NotImplementedError()
729 @abstractmethod
730 def getDeferred(
731 self,
732 datasetRefOrType: DatasetRef | DatasetType | str,
733 /,
734 dataId: DataId | None = None,
735 *,
736 parameters: dict | None = None,
737 collections: Any = None,
738 storageClass: str | StorageClass | None = None,
739 timespan: Timespan | None = None,
740 **kwargs: Any,
741 ) -> DeferredDatasetHandle:
742 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
743 after an immediate registry lookup.
745 Parameters
746 ----------
747 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
748 When `DatasetRef` the `dataId` should be `None`.
749 Otherwise the `DatasetType` or name thereof.
750 dataId : `dict` or `DataCoordinate`, optional
751 A `dict` of `Dimension` link name, value pairs that label the
752 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
753 should be provided as the first argument.
754 parameters : `dict`
755 Additional StorageClass-defined options to control reading,
756 typically used to efficiently read only a subset of the dataset.
757 collections : Any, optional
758 Collections to be searched, overriding ``self.collections``.
759 Can be any of the types supported by the ``collections`` argument
760 to butler construction.
761 storageClass : `StorageClass` or `str`, optional
762 The storage class to be used to override the Python type
763 returned by this method. By default the returned type matches
764 the dataset type definition for this dataset. Specifying a
765 read `StorageClass` can force a different type to be returned.
766 This type must be compatible with the original type.
767 timespan : `Timespan` or `None`, optional
768 A timespan that the validity range of the dataset must overlap.
769 If not provided and this is a calibration dataset type, an attempt
770 will be made to find the timespan from any temporal coordinate
771 in the data ID.
772 **kwargs
773 Additional keyword arguments used to augment or construct a
774 `DataId`. See `DataId` parameters.
776 Returns
777 -------
778 obj : `DeferredDatasetHandle`
779 A handle which can be used to retrieve a dataset at a later time.
781 Raises
782 ------
783 LookupError
784 Raised if no matching dataset exists in the `Registry` or
785 datastore.
786 ValueError
787 Raised if a resolved `DatasetRef` was passed as an input, but it
788 differs from the one found in the registry.
789 TypeError
790 Raised if no collections were provided.
791 """
792 raise NotImplementedError()
794 @abstractmethod
795 def get(
796 self,
797 datasetRefOrType: DatasetRef | DatasetType | str,
798 /,
799 dataId: DataId | None = None,
800 *,
801 parameters: dict[str, Any] | None = None,
802 collections: Any = None,
803 storageClass: StorageClass | str | None = None,
804 timespan: Timespan | None = None,
805 **kwargs: Any,
806 ) -> Any:
807 """Retrieve a stored dataset.
809 Parameters
810 ----------
811 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
812 When `DatasetRef` the `dataId` should be `None`.
813 Otherwise the `DatasetType` or name thereof.
814 If a resolved `DatasetRef`, the associated dataset
815 is returned directly without additional querying.
816 dataId : `dict` or `DataCoordinate`
817 A `dict` of `Dimension` link name, value pairs that label the
818 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
819 should be provided as the first argument.
820 parameters : `dict`
821 Additional StorageClass-defined options to control reading,
822 typically used to efficiently read only a subset of the dataset.
823 collections : Any, optional
824 Collections to be searched, overriding ``self.collections``.
825 Can be any of the types supported by the ``collections`` argument
826 to butler construction.
827 storageClass : `StorageClass` or `str`, optional
828 The storage class to be used to override the Python type
829 returned by this method. By default the returned type matches
830 the dataset type definition for this dataset. Specifying a
831 read `StorageClass` can force a different type to be returned.
832 This type must be compatible with the original type.
833 timespan : `Timespan` or `None`, optional
834 A timespan that the validity range of the dataset must overlap.
835 If not provided and this is a calibration dataset type, an attempt
836 will be made to find the timespan from any temporal coordinate
837 in the data ID.
838 **kwargs
839 Additional keyword arguments used to augment or construct a
840 `DataCoordinate`. See `DataCoordinate.standardize`
841 parameters.
843 Returns
844 -------
845 obj : `object`
846 The dataset.
848 Raises
849 ------
850 LookupError
851 Raised if no matching dataset exists in the `Registry`.
852 TypeError
853 Raised if no collections were provided.
855 Notes
856 -----
857 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
858 this method requires that the given data ID include temporal dimensions
859 beyond the dimensions of the dataset type itself, in order to find the
860 dataset with the appropriate validity range. For example, a "bias"
861 dataset with native dimensions ``{instrument, detector}`` could be
862 fetched with a ``{instrument, detector, exposure}`` data ID, because
863 ``exposure`` is a temporal dimension.
864 """
865 raise NotImplementedError()
867 @abstractmethod
868 def getURIs(
869 self,
870 datasetRefOrType: DatasetRef | DatasetType | str,
871 /,
872 dataId: DataId | None = None,
873 *,
874 predict: bool = False,
875 collections: Any = None,
876 run: str | None = None,
877 **kwargs: Any,
878 ) -> DatasetRefURIs:
879 """Return the URIs associated with the dataset.
881 Parameters
882 ----------
883 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
884 When `DatasetRef` the `dataId` should be `None`.
885 Otherwise the `DatasetType` or name thereof.
886 dataId : `dict` or `DataCoordinate`
887 A `dict` of `Dimension` link name, value pairs that label the
888 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
889 should be provided as the first argument.
890 predict : `bool`
891 If `True`, allow URIs to be returned of datasets that have not
892 been written.
893 collections : Any, optional
894 Collections to be searched, overriding ``self.collections``.
895 Can be any of the types supported by the ``collections`` argument
896 to butler construction.
897 run : `str`, optional
898 Run to use for predictions, overriding ``self.run``.
899 **kwargs
900 Additional keyword arguments used to augment or construct a
901 `DataCoordinate`. See `DataCoordinate.standardize`
902 parameters.
904 Returns
905 -------
906 uris : `DatasetRefURIs`
907 The URI to the primary artifact associated with this dataset (if
908 the dataset was disassembled within the datastore this may be
909 `None`), and the URIs to any components associated with the dataset
910 artifact. (can be empty if there are no components).
911 """
912 raise NotImplementedError()
914 def getURI(
915 self,
916 datasetRefOrType: DatasetRef | DatasetType | str,
917 /,
918 dataId: DataId | None = None,
919 *,
920 predict: bool = False,
921 collections: Any = None,
922 run: str | None = None,
923 **kwargs: Any,
924 ) -> ResourcePath:
925 """Return the URI to the Dataset.
927 Parameters
928 ----------
929 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
930 When `DatasetRef` the `dataId` should be `None`.
931 Otherwise the `DatasetType` or name thereof.
932 dataId : `dict` or `DataCoordinate`
933 A `dict` of `Dimension` link name, value pairs that label the
934 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
935 should be provided as the first argument.
936 predict : `bool`
937 If `True`, allow URIs to be returned of datasets that have not
938 been written.
939 collections : Any, optional
940 Collections to be searched, overriding ``self.collections``.
941 Can be any of the types supported by the ``collections`` argument
942 to butler construction.
943 run : `str`, optional
944 Run to use for predictions, overriding ``self.run``.
945 **kwargs
946 Additional keyword arguments used to augment or construct a
947 `DataCoordinate`. See `DataCoordinate.standardize`
948 parameters.
950 Returns
951 -------
952 uri : `lsst.resources.ResourcePath`
953 URI pointing to the Dataset within the datastore. If the
954 Dataset does not exist in the datastore, and if ``predict`` is
955 `True`, the URI will be a prediction and will include a URI
956 fragment "#predicted".
957 If the datastore does not have entities that relate well
958 to the concept of a URI the returned URI string will be
959 descriptive. The returned URI is not guaranteed to be obtainable.
961 Raises
962 ------
963 LookupError
964 A URI has been requested for a dataset that does not exist and
965 guessing is not allowed.
966 ValueError
967 Raised if a resolved `DatasetRef` was passed as an input, but it
968 differs from the one found in the registry.
969 TypeError
970 Raised if no collections were provided.
971 RuntimeError
972 Raised if a URI is requested for a dataset that consists of
973 multiple artifacts.
974 """
975 primary, components = self.getURIs(
976 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
977 )
979 if primary is None or components:
980 raise RuntimeError(
981 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
982 "Use Butler.getURIs() instead."
983 )
984 return primary
986 @abstractmethod
987 def get_dataset_type(self, name: str) -> DatasetType:
988 """Get the `DatasetType`.
990 Parameters
991 ----------
992 name : `str`
993 Name of the type.
995 Returns
996 -------
997 type : `DatasetType`
998 The `DatasetType` associated with the given name.
1000 Raises
1001 ------
1002 lsst.daf.butler.MissingDatasetTypeError
1003 Raised if the requested dataset type has not been registered.
1005 Notes
1006 -----
1007 This method handles component dataset types automatically, though most
1008 other operations do not.
1009 """
1010 raise NotImplementedError()
1012 @abstractmethod
1013 def get_dataset(
1014 self,
1015 id: DatasetId,
1016 *,
1017 storage_class: str | StorageClass | None = None,
1018 dimension_records: bool = False,
1019 datastore_records: bool = False,
1020 ) -> DatasetRef | None:
1021 """Retrieve a Dataset entry.
1023 Parameters
1024 ----------
1025 id : `DatasetId`
1026 The unique identifier for the dataset.
1027 storage_class : `str` or `StorageClass` or `None`
1028 A storage class to use when creating the returned entry. If given
1029 it must be compatible with the default storage class.
1030 dimension_records : `bool`, optional
1031 If `True` the ref will be expanded and contain dimension records.
1032 datastore_records : `bool`, optional
1033 If `True` the ref will contain associated datastore records.
1035 Returns
1036 -------
1037 ref : `DatasetRef` or `None`
1038 A ref to the Dataset, or `None` if no matching Dataset
1039 was found.
1040 """
1041 raise NotImplementedError()
1043 @abstractmethod
1044 def find_dataset(
1045 self,
1046 dataset_type: DatasetType | str,
1047 data_id: DataId | None = None,
1048 *,
1049 collections: str | Sequence[str] | None = None,
1050 timespan: Timespan | None = None,
1051 storage_class: str | StorageClass | None = None,
1052 dimension_records: bool = False,
1053 datastore_records: bool = False,
1054 **kwargs: Any,
1055 ) -> DatasetRef | None:
1056 """Find a dataset given its `DatasetType` and data ID.
1058 This can be used to obtain a `DatasetRef` that permits the dataset to
1059 be read from a `Datastore`. If the dataset is a component and can not
1060 be found using the provided dataset type, a dataset ref for the parent
1061 will be returned instead but with the correct dataset type.
1063 Parameters
1064 ----------
1065 dataset_type : `DatasetType` or `str`
1066 A `DatasetType` or the name of one. If this is a `DatasetType`
1067 instance, its storage class will be respected and propagated to
1068 the output, even if it differs from the dataset type definition
1069 in the registry, as long as the storage classes are convertible.
1070 data_id : `dict` or `DataCoordinate`, optional
1071 A `dict`-like object containing the `Dimension` links that identify
1072 the dataset within a collection. If it is a `dict` the dataId
1073 can include dimension record values such as ``day_obs`` and
1074 ``seq_num`` or ``full_name`` that can be used to derive the
1075 primary dimension.
1076 collections : `str` or `list` [`str`], optional
1077 A an ordered list of collections to search for the dataset.
1078 Defaults to ``self.defaults.collections``.
1079 timespan : `Timespan`, optional
1080 A timespan that the validity range of the dataset must overlap.
1081 If not provided, any `~CollectionType.CALIBRATION` collections
1082 matched by the ``collections`` argument will not be searched.
1083 storage_class : `str` or `StorageClass` or `None`
1084 A storage class to use when creating the returned entry. If given
1085 it must be compatible with the default storage class.
1086 dimension_records : `bool`, optional
1087 If `True` the ref will be expanded and contain dimension records.
1088 datastore_records : `bool`, optional
1089 If `True` the ref will contain associated datastore records.
1090 **kwargs
1091 Additional keyword arguments passed to
1092 `DataCoordinate.standardize` to convert ``dataId`` to a true
1093 `DataCoordinate` or augment an existing one. This can also include
1094 dimension record metadata that can be used to derive a primary
1095 dimension value.
1097 Returns
1098 -------
1099 ref : `DatasetRef`
1100 A reference to the dataset, or `None` if no matching Dataset
1101 was found.
1103 Raises
1104 ------
1105 lsst.daf.butler.NoDefaultCollectionError
1106 Raised if ``collections`` is `None` and
1107 ``self.collections`` is `None`.
1108 LookupError
1109 Raised if one or more data ID keys are missing.
1110 lsst.daf.butler.MissingDatasetTypeError
1111 Raised if the dataset type does not exist.
1112 lsst.daf.butler.MissingCollectionError
1113 Raised if any of ``collections`` does not exist in the registry.
1115 Notes
1116 -----
1117 This method simply returns `None` and does not raise an exception even
1118 when the set of collections searched is intrinsically incompatible with
1119 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1120 only `~CollectionType.CALIBRATION` collections are being searched.
1121 This may make it harder to debug some lookup failures, but the behavior
1122 is intentional; we consider it more important that failed searches are
1123 reported consistently, regardless of the reason, and that adding
1124 additional collections that do not contain a match to the search path
1125 never changes the behavior.
1127 This method handles component dataset types automatically, though most
1128 other query operations do not.
1129 """
1130 raise NotImplementedError()
1132 @abstractmethod
1133 def retrieve_artifacts_zip(
1134 self,
1135 refs: Iterable[DatasetRef],
1136 destination: ResourcePathExpression,
1137 overwrite: bool = True,
1138 ) -> ResourcePath:
1139 """Retrieve artifacts from a Butler and place in ZIP file.
1141 Parameters
1142 ----------
1143 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1144 The datasets to be included in the zip file.
1145 destination : `lsst.resources.ResourcePathExpression`
1146 Directory to write the new ZIP file. This directory will
1147 also be used as a staging area for the datasets being downloaded
1148 from the datastore.
1149 overwrite : `bool`, optional
1150 If `False` the output Zip will not be written if a file of the
1151 same name is already present in ``destination``.
1153 Returns
1154 -------
1155 zip_file : `lsst.resources.ResourcePath`
1156 The path to the new ZIP file.
1158 Raises
1159 ------
1160 ValueError
1161 Raised if there are no refs to retrieve.
1162 """
1163 raise NotImplementedError()
1165 @abstractmethod
1166 def retrieveArtifacts(
1167 self,
1168 refs: Iterable[DatasetRef],
1169 destination: ResourcePathExpression,
1170 transfer: str = "auto",
1171 preserve_path: bool = True,
1172 overwrite: bool = False,
1173 ) -> list[ResourcePath]:
1174 """Retrieve the artifacts associated with the supplied refs.
1176 Parameters
1177 ----------
1178 refs : iterable of `DatasetRef`
1179 The datasets for which artifacts are to be retrieved.
1180 A single ref can result in multiple artifacts. The refs must
1181 be resolved.
1182 destination : `lsst.resources.ResourcePath` or `str`
1183 Location to write the artifacts.
1184 transfer : `str`, optional
1185 Method to use to transfer the artifacts. Must be one of the options
1186 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1187 "move" is not allowed.
1188 preserve_path : `bool`, optional
1189 If `True` the full path of the artifact within the datastore
1190 is preserved. If `False` the final file component of the path
1191 is used.
1192 overwrite : `bool`, optional
1193 If `True` allow transfers to overwrite existing files at the
1194 destination.
1196 Returns
1197 -------
1198 targets : `list` of `lsst.resources.ResourcePath`
1199 URIs of file artifacts in destination location. Order is not
1200 preserved.
1202 Notes
1203 -----
1204 For non-file datastores the artifacts written to the destination
1205 may not match the representation inside the datastore. For example
1206 a hierarchical data structure in a NoSQL database may well be stored
1207 as a JSON file.
1208 """
1209 raise NotImplementedError()
1211 @abstractmethod
1212 def exists(
1213 self,
1214 dataset_ref_or_type: DatasetRef | DatasetType | str,
1215 /,
1216 data_id: DataId | None = None,
1217 *,
1218 full_check: bool = True,
1219 collections: Any = None,
1220 **kwargs: Any,
1221 ) -> DatasetExistence:
1222 """Indicate whether a dataset is known to Butler registry and
1223 datastore.
1225 Parameters
1226 ----------
1227 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1228 When `DatasetRef` the `dataId` should be `None`.
1229 Otherwise the `DatasetType` or name thereof.
1230 data_id : `dict` or `DataCoordinate`
1231 A `dict` of `Dimension` link name, value pairs that label the
1232 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1233 should be provided as the first argument.
1234 full_check : `bool`, optional
1235 If `True`, a check will be made for the actual existence of a
1236 dataset artifact. This will involve additional overhead due to
1237 the need to query an external system. If `False`, this check will
1238 be omitted, and the registry and datastore will solely be asked
1239 if they know about the dataset but no direct check for the
1240 artifact will be performed.
1241 collections : Any, optional
1242 Collections to be searched, overriding ``self.collections``.
1243 Can be any of the types supported by the ``collections`` argument
1244 to butler construction.
1245 **kwargs
1246 Additional keyword arguments used to augment or construct a
1247 `DataCoordinate`. See `DataCoordinate.standardize`
1248 parameters.
1250 Returns
1251 -------
1252 existence : `DatasetExistence`
1253 Object indicating whether the dataset is known to registry and
1254 datastore. Evaluates to `True` if the dataset is present and known
1255 to both.
1256 """
1257 raise NotImplementedError()
1259 @abstractmethod
1260 def _exists_many(
1261 self,
1262 refs: Iterable[DatasetRef],
1263 /,
1264 *,
1265 full_check: bool = True,
1266 ) -> dict[DatasetRef, DatasetExistence]:
1267 """Indicate whether multiple datasets are known to Butler registry and
1268 datastore.
1270 This is an experimental API that may change at any moment.
1272 Parameters
1273 ----------
1274 refs : iterable of `DatasetRef`
1275 The datasets to be checked.
1276 full_check : `bool`, optional
1277 If `True`, a check will be made for the actual existence of each
1278 dataset artifact. This will involve additional overhead due to
1279 the need to query an external system. If `False`, this check will
1280 be omitted, and the registry and datastore will solely be asked
1281 if they know about the dataset(s) but no direct check for the
1282 artifact(s) will be performed.
1284 Returns
1285 -------
1286 existence : dict of [`DatasetRef`, `DatasetExistence`]
1287 Mapping from the given dataset refs to an enum indicating the
1288 status of the dataset in registry and datastore.
1289 Each value evaluates to `True` if the dataset is present and known
1290 to both.
1291 """
1292 raise NotImplementedError()
1294 @abstractmethod
1295 def removeRuns(
1296 self,
1297 names: Iterable[str],
1298 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1299 *,
1300 unlink_from_chains: bool = False,
1301 ) -> None:
1302 """Remove one or more `~CollectionType.RUN` collections and the
1303 datasets within them.
1305 Parameters
1306 ----------
1307 names : `~collections.abc.Iterable` [ `str` ]
1308 The names of the collections to remove.
1309 unstore : `bool`, optional
1310 If `True` (default), delete datasets from all datastores in which
1311 they are present, and attempt to rollback the registry deletions if
1312 datastore deletions fail (which may not always be possible). If
1313 `False`, datastore records for these datasets are still removed,
1314 but any artifacts (e.g. files) will not be. This parameter is now
1315 deprecated and no longer has any effect. Files are always deleted
1316 from datastores unless they were ingested using full URIs.
1317 unlink_from_chains : `bool`, optional
1318 If `True` remove the RUN collection from any chains prior to
1319 removing the RUN. If `False` the removal will fail if any chains
1320 still refer to the RUN.
1322 Raises
1323 ------
1324 TypeError
1325 Raised if one or more collections are not of type
1326 `~CollectionType.RUN`.
1327 """
1328 raise NotImplementedError()
1330 @abstractmethod
1331 def ingest(
1332 self,
1333 *datasets: FileDataset,
1334 transfer: str | None = "auto",
1335 record_validation_info: bool = True,
1336 skip_existing: bool = False,
1337 ) -> None:
1338 """Store and register one or more datasets that already exist on disk.
1340 Parameters
1341 ----------
1342 *datasets : `FileDataset`
1343 Each positional argument is a struct containing information about
1344 a file to be ingested, including its URI (either absolute or
1345 relative to the datastore root, if applicable), a resolved
1346 `DatasetRef`, and optionally a formatter class or its
1347 fully-qualified string name. If a formatter is not provided, the
1348 formatter that would be used for `put` is assumed. On successful
1349 ingest all `FileDataset.formatter` attributes will be set to the
1350 formatter class used. `FileDataset.path` attributes may be modified
1351 to put paths in whatever the datastore considers a standardized
1352 form.
1353 transfer : `str`, optional
1354 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1355 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1356 transfer the file.
1357 record_validation_info : `bool`, optional
1358 If `True`, the default, the datastore can record validation
1359 information associated with the file. If `False` the datastore
1360 will not attempt to track any information such as checksums
1361 or file sizes. This can be useful if such information is tracked
1362 in an external system or if the file is to be compressed in place.
1363 It is up to the datastore whether this parameter is relevant.
1364 skip_existing : `bool`, optional
1365 If `True`, a dataset will not be ingested if a dataset with the
1366 same dataset ID already exists in the datastore.
1367 If `False` (the default), a `ConflictingDefinitionError` will be
1368 raised if any datasets with the same dataset ID already exist
1369 in the datastore.
1371 Raises
1372 ------
1373 TypeError
1374 Raised if the butler is read-only or if no run was provided.
1375 NotImplementedError
1376 Raised if the `Datastore` does not support the given transfer mode.
1377 DatasetTypeNotSupportedError
1378 Raised if one or more files to be ingested have a dataset type that
1379 is not supported by the `Datastore`..
1380 FileNotFoundError
1381 Raised if one of the given files does not exist.
1382 FileExistsError
1383 Raised if transfer is not `None` but the (internal) location the
1384 file would be moved to is already occupied.
1385 ConflictingDefinitionError
1386 Raised if a dataset already exists in the repository and
1387 ``skip_existing`` is `False`.
1389 Notes
1390 -----
1391 This operation is not fully exception safe: if a database operation
1392 fails, the given `FileDataset` instances may be only partially updated.
1394 It is atomic in terms of database operations (they will either all
1395 succeed or all fail) providing the database engine implements
1396 transactions correctly. It will attempt to be atomic in terms of
1397 filesystem operations as well, but this cannot be implemented
1398 rigorously for most datastores.
1399 """
1400 raise NotImplementedError()
1402 @abstractmethod
1403 def ingest_zip(
1404 self,
1405 zip_file: ResourcePathExpression,
1406 transfer: str = "auto",
1407 *,
1408 transfer_dimensions: bool = False,
1409 dry_run: bool = False,
1410 ) -> None:
1411 """Ingest a Zip file into this butler.
1413 The Zip file must have been created by `retrieve_artifacts_zip`.
1415 Parameters
1416 ----------
1417 zip_file : `lsst.resources.ResourcePathExpression`
1418 Path to the Zip file.
1419 transfer : `str`, optional
1420 Method to use to transfer the Zip into the datastore.
1421 transfer_dimensions : `bool`, optional
1422 If `True`, dimension record data associated with the new datasets
1423 will be transferred from the Zip file, if present.
1424 dry_run : `bool`, optional
1425 If `True` the ingest will be processed without any modifications
1426 made to the target butler and as if the target butler did not
1427 have any of the datasets.
1429 Notes
1430 -----
1431 Run collections and dataset types are created as needed.
1432 """
1433 raise NotImplementedError()
1435 @abstractmethod
1436 def export(
1437 self,
1438 *,
1439 directory: str | None = None,
1440 filename: str | None = None,
1441 format: str | None = None,
1442 transfer: str | None = None,
1443 ) -> AbstractContextManager[RepoExportContext]:
1444 """Export datasets from the repository represented by this `Butler`.
1446 This method is a context manager that returns a helper object
1447 (`RepoExportContext`) that is used to indicate what information from
1448 the repository should be exported.
1450 Parameters
1451 ----------
1452 directory : `str`, optional
1453 Directory dataset files should be written to if ``transfer`` is not
1454 `None`.
1455 filename : `str`, optional
1456 Name for the file that will include database information associated
1457 with the exported datasets. If this is not an absolute path and
1458 ``directory`` is not `None`, it will be written to ``directory``
1459 instead of the current working directory. Defaults to
1460 "export.{format}".
1461 format : `str`, optional
1462 File format for the database information file. If `None`, the
1463 extension of ``filename`` will be used.
1464 transfer : `str`, optional
1465 Transfer mode passed to `Datastore.export`.
1467 Raises
1468 ------
1469 TypeError
1470 Raised if the set of arguments passed is inconsistent.
1472 Examples
1473 --------
1474 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1475 methods are used to provide the iterables over data IDs and/or datasets
1476 to be exported::
1478 with butler.export("exports.yaml") as export:
1479 # Export all flats, but none of the dimension element rows
1480 # (i.e. data ID information) associated with them.
1481 export.saveDatasets(
1482 butler.registry.queryDatasets("flat"), elements=()
1483 )
1484 # Export all datasets that start with "deepCoadd_" and all of
1485 # their associated data ID information.
1486 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1487 """
1488 raise NotImplementedError()
1490 @abstractmethod
1491 def import_(
1492 self,
1493 *,
1494 directory: ResourcePathExpression | None = None,
1495 filename: ResourcePathExpression | TextIO | None = None,
1496 format: str | None = None,
1497 transfer: str | None = None,
1498 skip_dimensions: set | None = None,
1499 record_validation_info: bool = True,
1500 without_datastore: bool = False,
1501 ) -> None:
1502 """Import datasets into this repository that were exported from a
1503 different butler repository via `~lsst.daf.butler.Butler.export`.
1505 Parameters
1506 ----------
1507 directory : `~lsst.resources.ResourcePathExpression`, optional
1508 Directory containing dataset files to import from. If `None`,
1509 ``filename`` and all dataset file paths specified therein must
1510 be absolute.
1511 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1512 A stream or name of file that contains database information
1513 associated with the exported datasets, typically generated by
1514 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1515 `~lsst.resources.ResourcePath` and is not an absolute path,
1516 it will first be looked for relative to ``directory`` and if not
1517 found there it will be looked for in the current working
1518 directory. Defaults to "export.{format}".
1519 format : `str`, optional
1520 File format for ``filename``. If `None`, the extension of
1521 ``filename`` will be used.
1522 transfer : `str`, optional
1523 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1524 skip_dimensions : `set`, optional
1525 Names of dimensions that should be skipped and not imported.
1526 record_validation_info : `bool`, optional
1527 If `True`, the default, the datastore can record validation
1528 information associated with the file. If `False` the datastore
1529 will not attempt to track any information such as checksums
1530 or file sizes. This can be useful if such information is tracked
1531 in an external system or if the file is to be compressed in place.
1532 It is up to the datastore whether this parameter is relevant.
1533 without_datastore : `bool`, optional
1534 If `True` only registry records will be imported and the datastore
1535 will be ignored.
1537 Raises
1538 ------
1539 TypeError
1540 Raised if the set of arguments passed is inconsistent, or if the
1541 butler is read-only.
1542 """
1543 raise NotImplementedError()
1545 @abstractmethod
1546 def transfer_dimension_records_from(
1547 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef]
1548 ) -> None:
1549 """Transfer dimension records to this Butler from another Butler.
1551 Parameters
1552 ----------
1553 source_butler : `LimitedButler` or `Butler`
1554 Butler from which the records are to be transferred. If data IDs
1555 in ``source_refs`` are not expanded then this has to be a full
1556 `Butler` whose registry will be used to expand data IDs. If the
1557 source refs contain coordinates that are used to populate other
1558 records then this will also need to be a full `Butler`.
1559 source_refs : iterable of `DatasetRef`
1560 Datasets defined in the source butler whose dimension records
1561 should be transferred to this butler. In most circumstances.
1562 transfer is faster if the dataset refs are expanded.
1563 """
1564 raise NotImplementedError()
1566 @abstractmethod
1567 def transfer_from(
1568 self,
1569 source_butler: LimitedButler,
1570 source_refs: Iterable[DatasetRef],
1571 transfer: str = "auto",
1572 skip_missing: bool = True,
1573 register_dataset_types: bool = False,
1574 transfer_dimensions: bool = False,
1575 dry_run: bool = False,
1576 ) -> Collection[DatasetRef]:
1577 """Transfer datasets to this Butler from a run in another Butler.
1579 Parameters
1580 ----------
1581 source_butler : `LimitedButler`
1582 Butler from which the datasets are to be transferred. If data IDs
1583 in ``source_refs`` are not expanded then this has to be a full
1584 `Butler` whose registry will be used to expand data IDs.
1585 source_refs : iterable of `DatasetRef`
1586 Datasets defined in the source butler that should be transferred to
1587 this butler. In most circumstances, ``transfer_from`` is faster if
1588 the dataset refs are expanded.
1589 transfer : `str`, optional
1590 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1591 skip_missing : `bool`
1592 If `True`, datasets with no datastore artifact associated with
1593 them are not transferred. If `False` a registry entry will be
1594 created even if no datastore record is created (and so will
1595 look equivalent to the dataset being unstored).
1596 register_dataset_types : `bool`
1597 If `True` any missing dataset types are registered. Otherwise
1598 an exception is raised.
1599 transfer_dimensions : `bool`, optional
1600 If `True`, dimension record data associated with the new datasets
1601 will be transferred.
1602 dry_run : `bool`, optional
1603 If `True` the transfer will be processed without any modifications
1604 made to the target butler and as if the target butler did not
1605 have any of the datasets.
1607 Returns
1608 -------
1609 refs : `list` of `DatasetRef`
1610 The refs added to this Butler.
1612 Notes
1613 -----
1614 The datastore artifact has to exist for a transfer
1615 to be made but non-existence is not an error.
1617 Datasets that already exist in this run will be skipped.
1619 The datasets are imported as part of a transaction, although
1620 dataset types are registered before the transaction is started.
1621 This means that it is possible for a dataset type to be registered
1622 even though transfer has failed.
1623 """
1624 raise NotImplementedError()
1626 @abstractmethod
1627 def validateConfiguration(
1628 self,
1629 logFailures: bool = False,
1630 datasetTypeNames: Iterable[str] | None = None,
1631 ignore: Iterable[str] | None = None,
1632 ) -> None:
1633 """Validate butler configuration.
1635 Checks that each `DatasetType` can be stored in the `Datastore`.
1637 Parameters
1638 ----------
1639 logFailures : `bool`, optional
1640 If `True`, output a log message for every validation error
1641 detected.
1642 datasetTypeNames : iterable of `str`, optional
1643 The `DatasetType` names that should be checked. This allows
1644 only a subset to be selected.
1645 ignore : iterable of `str`, optional
1646 Names of DatasetTypes to skip over. This can be used to skip
1647 known problems. If a named `DatasetType` corresponds to a
1648 composite, all components of that `DatasetType` will also be
1649 ignored.
1651 Raises
1652 ------
1653 ButlerValidationError
1654 Raised if there is some inconsistency with how this Butler
1655 is configured.
1656 """
1657 raise NotImplementedError()
1659 @property
1660 @abstractmethod
1661 def collection_chains(self) -> ButlerCollections:
1662 """Object with methods for modifying collection chains
1663 (`~lsst.daf.butler.ButlerCollections`).
1665 Deprecated. Replaced with ``collections`` property.
1666 """
1667 raise NotImplementedError()
1669 @property
1670 @abstractmethod
1671 def collections(self) -> ButlerCollections:
1672 """Object with methods for modifying and querying collections
1673 (`~lsst.daf.butler.ButlerCollections`).
1675 Use of this object is preferred over `registry` wherever possible.
1676 """
1677 raise NotImplementedError()
1679 @property
1680 @abstractmethod
1681 def run(self) -> str | None:
1682 """Name of the run this butler writes outputs to by default (`str` or
1683 `None`).
1684 """
1685 raise NotImplementedError()
1687 @property
1688 @abstractmethod
1689 def registry(self) -> Registry:
1690 """The object that manages dataset metadata and relationships
1691 (`Registry`).
1693 Many operations that don't involve reading or writing butler datasets
1694 are accessible only via `Registry` methods. Eventually these methods
1695 will be replaced by equivalent `Butler` methods.
1696 """
1697 raise NotImplementedError()
1699 @abstractmethod
1700 def query(self) -> AbstractContextManager[Query]:
1701 """Context manager returning a `Query` object used for construction
1702 and execution of complex queries.
1703 """
1704 raise NotImplementedError()
1706 def query_data_ids(
1707 self,
1708 dimensions: DimensionGroup | Iterable[str] | str,
1709 *,
1710 data_id: DataId | None = None,
1711 where: str = "",
1712 bind: Mapping[str, Any] | None = None,
1713 with_dimension_records: bool = False,
1714 order_by: Iterable[str] | str | None = None,
1715 limit: int | None = -20_000,
1716 explain: bool = True,
1717 **kwargs: Any,
1718 ) -> list[DataCoordinate]:
1719 """Query for data IDs matching user-provided criteria.
1721 Parameters
1722 ----------
1723 dimensions : `DimensionGroup`, `str`, or \
1724 `~collections.abc.Iterable` [`str`]
1725 The dimensions of the data IDs to yield, as either `DimensionGroup`
1726 instances or `str`. Will be automatically expanded to a complete
1727 `DimensionGroup`.
1728 data_id : `dict` or `DataCoordinate`, optional
1729 A data ID whose key-value pairs are used as equality constraints
1730 in the query.
1731 where : `str`, optional
1732 A string expression similar to a SQL WHERE clause. May involve
1733 any column of a dimension table or (as a shortcut for the primary
1734 key column of a dimension table) dimension name. See
1735 :ref:`daf_butler_dimension_expressions` for more information.
1736 bind : `~collections.abc.Mapping`, optional
1737 Mapping containing literal values that should be injected into the
1738 ``where`` expression, keyed by the identifiers they replace.
1739 Values of collection type can be expanded in some cases; see
1740 :ref:`daf_butler_dimension_expressions_identifiers` for more
1741 information.
1742 with_dimension_records : `bool`, optional
1743 If `True` (default is `False`) then returned data IDs will have
1744 dimension records.
1745 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1746 Names of the columns/dimensions to use for ordering returned data
1747 IDs. Column name can be prefixed with minus (``-``) to use
1748 descending ordering.
1749 limit : `int` or `None`, optional
1750 Upper limit on the number of returned records. `None` can be used
1751 if no limit is wanted. A limit of ``0`` means that the query will
1752 be executed and validated but no results will be returned. In this
1753 case there will be no exception even if ``explain`` is `True`.
1754 If a negative value is given a warning will be issued if the number
1755 of results is capped by that limit.
1756 explain : `bool`, optional
1757 If `True` (default) then `EmptyQueryResultError` exception is
1758 raised when resulting list is empty. The exception contains
1759 non-empty list of strings explaining possible causes for empty
1760 result.
1761 **kwargs
1762 Additional keyword arguments are forwarded to
1763 `DataCoordinate.standardize` when processing the ``data_id``
1764 argument (and may be used to provide a constraining data ID even
1765 when the ``data_id`` argument is `None`).
1767 Returns
1768 -------
1769 dataIds : `list` [`DataCoordinate`]
1770 Data IDs matching the given query parameters. These are always
1771 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1772 returns `True`).
1774 Raises
1775 ------
1776 lsst.daf.butler.registry.DataIdError
1777 Raised when ``data_id`` or keyword arguments specify unknown
1778 dimensions or values, or when they contain inconsistent values.
1779 lsst.daf.butler.registry.UserExpressionError
1780 Raised when ``where`` expression is invalid.
1781 lsst.daf.butler.EmptyQueryResultError
1782 Raised when query generates empty result and ``explain`` is set to
1783 `True`.
1784 TypeError
1785 Raised when the arguments are incompatible.
1786 """
1787 if data_id is None:
1788 data_id = DataCoordinate.make_empty(self.dimensions)
1789 if order_by is None:
1790 order_by = []
1791 query_limit = limit
1792 warn_limit = False
1793 if limit is not None and limit < 0:
1794 query_limit = abs(limit) + 1
1795 warn_limit = True
1796 with self.query() as query:
1797 result = (
1798 query.data_ids(dimensions)
1799 .where(data_id, where, bind=bind, **kwargs)
1800 .order_by(*ensure_iterable(order_by))
1801 .limit(query_limit)
1802 )
1803 if with_dimension_records:
1804 result = result.with_dimension_records()
1805 data_ids = list(result)
1806 if warn_limit and len(data_ids) == query_limit:
1807 # We asked for one too many so must remove that from the list.
1808 data_ids.pop(-1)
1809 assert limit is not None # For mypy.
1810 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1811 if explain and (limit is None or limit != 0) and not data_ids:
1812 raise EmptyQueryResultError(list(result.explain_no_results()))
1813 return data_ids
1815 def query_datasets(
1816 self,
1817 dataset_type: str | DatasetType,
1818 collections: str | Iterable[str] | None = None,
1819 *,
1820 find_first: bool = True,
1821 data_id: DataId | None = None,
1822 where: str = "",
1823 bind: Mapping[str, Any] | None = None,
1824 with_dimension_records: bool = False,
1825 order_by: Iterable[str] | str | None = None,
1826 limit: int | None = -20_000,
1827 explain: bool = True,
1828 **kwargs: Any,
1829 ) -> list[DatasetRef]:
1830 """Query for dataset references matching user-provided criteria.
1832 Parameters
1833 ----------
1834 dataset_type : `str` or `DatasetType`
1835 Dataset type object or name to search for.
1836 collections : collection expression, optional
1837 A collection name or iterable of collection names to search. If not
1838 provided, the default collections are used. Can be a wildcard if
1839 ``find_first`` is `False` (if find first is requested the order
1840 of collections matters and wildcards make the order indeterminate).
1841 See :ref:`daf_butler_collection_expressions` for more information.
1842 find_first : `bool`, optional
1843 If `True` (default), for each result data ID, only yield one
1844 `DatasetRef` of each `DatasetType`, from the first collection in
1845 which a dataset of that dataset type appears (according to the
1846 order of ``collections`` passed in). If `True`, ``collections``
1847 must not contain wildcards.
1848 data_id : `dict` or `DataCoordinate`, optional
1849 A data ID whose key-value pairs are used as equality constraints in
1850 the query.
1851 where : `str`, optional
1852 A string expression similar to a SQL WHERE clause. May involve any
1853 column of a dimension table or (as a shortcut for the primary key
1854 column of a dimension table) dimension name. See
1855 :ref:`daf_butler_dimension_expressions` for more information.
1856 bind : `~collections.abc.Mapping`, optional
1857 Mapping containing literal values that should be injected into the
1858 ``where`` expression, keyed by the identifiers they replace. Values
1859 of collection type can be expanded in some cases; see
1860 :ref:`daf_butler_dimension_expressions_identifiers` for more
1861 information.
1862 with_dimension_records : `bool`, optional
1863 If `True` (default is `False`) then returned data IDs will have
1864 dimension records.
1865 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1866 Names of the columns/dimensions to use for ordering returned data
1867 IDs. Column name can be prefixed with minus (``-``) to use
1868 descending ordering.
1869 limit : `int` or `None`, optional
1870 Upper limit on the number of returned records. `None` can be used
1871 if no limit is wanted. A limit of ``0`` means that the query will
1872 be executed and validated but no results will be returned. In this
1873 case there will be no exception even if ``explain`` is `True`.
1874 If a negative value is given a warning will be issued if the number
1875 of results is capped by that limit.
1876 explain : `bool`, optional
1877 If `True` (default) then `EmptyQueryResultError` exception is
1878 raised when resulting list is empty. The exception contains
1879 non-empty list of strings explaining possible causes for empty
1880 result.
1881 **kwargs
1882 Additional keyword arguments are forwarded to
1883 `DataCoordinate.standardize` when processing the ``data_id``
1884 argument (and may be used to provide a constraining data ID even
1885 when the ``data_id`` argument is `None`).
1887 Returns
1888 -------
1889 refs : `.queries.DatasetRefQueryResults`
1890 Dataset references matching the given query criteria. Nested data
1891 IDs are guaranteed to include values for all implied dimensions
1892 (i.e. `DataCoordinate.hasFull` will return `True`).
1894 Raises
1895 ------
1896 lsst.daf.butler.registry.DatasetTypeExpressionError
1897 Raised when ``dataset_type`` expression is invalid.
1898 lsst.daf.butler.registry.DataIdError
1899 Raised when ``data_id`` or keyword arguments specify unknown
1900 dimensions or values, or when they contain inconsistent values.
1901 lsst.daf.butler.registry.UserExpressionError
1902 Raised when ``where`` expression is invalid.
1903 lsst.daf.butler.EmptyQueryResultError
1904 Raised when query generates empty result and ``explain`` is set to
1905 `True`.
1906 TypeError
1907 Raised when the arguments are incompatible, such as when a
1908 collection wildcard is passed when ``find_first`` is `True`, or
1909 when ``collections`` is `None` and default butler collections are
1910 not defined.
1911 """
1912 if data_id is None:
1913 data_id = DataCoordinate.make_empty(self.dimensions)
1914 if order_by is None:
1915 order_by = []
1916 if collections and has_globs(collections):
1917 # Wild cards need to be expanded but can only be allowed if
1918 # find_first=False because expanding wildcards does not return
1919 # a guaranteed ordering. Querying collection registry to expand
1920 # collections when we do not have wildcards is expensive so only
1921 # do it if we need it.
1922 if find_first:
1923 raise InvalidQueryError(
1924 f"Can not use wildcards in collections when find_first=True (given {collections})"
1925 )
1926 collections = self.collections.query(collections)
1927 query_limit = limit
1928 warn_limit = False
1929 if limit is not None and limit < 0:
1930 query_limit = abs(limit) + 1
1931 warn_limit = True
1932 with self.query() as query:
1933 result = (
1934 query.datasets(dataset_type, collections=collections, find_first=find_first)
1935 .where(data_id, where, bind=bind, **kwargs)
1936 .order_by(*ensure_iterable(order_by))
1937 .limit(query_limit)
1938 )
1939 if with_dimension_records:
1940 result = result.with_dimension_records()
1941 refs = list(result)
1942 if warn_limit and len(refs) == query_limit:
1943 # We asked for one too many so must remove that from the list.
1944 refs.pop(-1)
1945 assert limit is not None # For mypy.
1946 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1947 if explain and (limit is None or limit != 0) and not refs:
1948 raise EmptyQueryResultError(list(result.explain_no_results()))
1949 return refs
1951 def query_dimension_records(
1952 self,
1953 element: str,
1954 *,
1955 data_id: DataId | None = None,
1956 where: str = "",
1957 bind: Mapping[str, Any] | None = None,
1958 order_by: Iterable[str] | str | None = None,
1959 limit: int | None = -20_000,
1960 explain: bool = True,
1961 **kwargs: Any,
1962 ) -> list[DimensionRecord]:
1963 """Query for dimension information matching user-provided criteria.
1965 Parameters
1966 ----------
1967 element : `str`
1968 The name of a dimension element to obtain records for.
1969 data_id : `dict` or `DataCoordinate`, optional
1970 A data ID whose key-value pairs are used as equality constraints
1971 in the query.
1972 where : `str`, optional
1973 A string expression similar to a SQL WHERE clause. See
1974 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1975 information.
1976 bind : `~collections.abc.Mapping`, optional
1977 Mapping containing literal values that should be injected into the
1978 ``where`` expression, keyed by the identifiers they replace.
1979 Values of collection type can be expanded in some cases; see
1980 :ref:`daf_butler_dimension_expressions_identifiers` for more
1981 information.
1982 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1983 Names of the columns/dimensions to use for ordering returned data
1984 IDs. Column name can be prefixed with minus (``-``) to use
1985 descending ordering.
1986 limit : `int` or `None`, optional
1987 Upper limit on the number of returned records. `None` can be used
1988 if no limit is wanted. A limit of ``0`` means that the query will
1989 be executed and validated but no results will be returned. In this
1990 case there will be no exception even if ``explain`` is `True`.
1991 If a negative value is given a warning will be issued if the number
1992 of results is capped by that limit.
1993 explain : `bool`, optional
1994 If `True` (default) then `EmptyQueryResultError` exception is
1995 raised when resulting list is empty. The exception contains
1996 non-empty list of strings explaining possible causes for empty
1997 result.
1998 **kwargs
1999 Additional keyword arguments are forwarded to
2000 `DataCoordinate.standardize` when processing the ``data_id``
2001 argument (and may be used to provide a constraining data ID even
2002 when the ``data_id`` argument is `None`).
2004 Returns
2005 -------
2006 records : `list`[`DimensionRecord`]
2007 Dimension records matching the given query parameters.
2009 Raises
2010 ------
2011 lsst.daf.butler.registry.DataIdError
2012 Raised when ``data_id`` or keyword arguments specify unknown
2013 dimensions or values, or when they contain inconsistent values.
2014 lsst.daf.butler.registry.UserExpressionError
2015 Raised when ``where`` expression is invalid.
2016 lsst.daf.butler.EmptyQueryResultError
2017 Raised when query generates empty result and ``explain`` is set to
2018 `True`.
2019 TypeError
2020 Raised when the arguments are incompatible, such as when a
2021 collection wildcard is passed when ``find_first`` is `True`, or
2022 when ``collections`` is `None` and default butler collections are
2023 not defined.
2024 """
2025 if data_id is None:
2026 data_id = DataCoordinate.make_empty(self.dimensions)
2027 if order_by is None:
2028 order_by = []
2029 query_limit = limit
2030 warn_limit = False
2031 if limit is not None and limit < 0:
2032 query_limit = abs(limit) + 1
2033 warn_limit = True
2034 with self.query() as query:
2035 result = (
2036 query.dimension_records(element)
2037 .where(data_id, where, bind=bind, **kwargs)
2038 .order_by(*ensure_iterable(order_by))
2039 .limit(query_limit)
2040 )
2041 dimension_records = list(result)
2042 if warn_limit and len(dimension_records) == query_limit:
2043 # We asked for one too many so must remove that from the list.
2044 dimension_records.pop(-1)
2045 assert limit is not None # For mypy.
2046 _LOG.warning(
2047 "More dimension records are available than the requested limit of %d.", abs(limit)
2048 )
2049 if explain and (limit is None or limit != 0) and not dimension_records:
2050 raise EmptyQueryResultError(list(result.explain_no_results()))
2051 return dimension_records
2053 def query_all_datasets(
2054 self,
2055 collections: str | Iterable[str] | None = None,
2056 *,
2057 name: str | Iterable[str] = "*",
2058 find_first: bool = True,
2059 data_id: DataId | None = None,
2060 where: str = "",
2061 bind: Mapping[str, Any] | None = None,
2062 limit: int | None = -20_000,
2063 **kwargs: Any,
2064 ) -> list[DatasetRef]:
2065 """Query for datasets of potentially multiple types.
2067 Parameters
2068 ----------
2069 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2070 The collection or collections to search, in order. If not provided
2071 or `None`, the default collection search path for this butler is
2072 used.
2073 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2074 Names or name patterns (glob-style) that returned dataset type
2075 names must match. If an iterable, items are OR'd together. The
2076 default is to include all dataset types in the given collections.
2077 find_first : `bool`, optional
2078 If `True` (default), for each result data ID, only yield one
2079 `DatasetRef` of each `DatasetType`, from the first collection in
2080 which a dataset of that dataset type appears (according to the
2081 order of ``collections`` passed in).
2082 data_id : `dict` or `DataCoordinate`, optional
2083 A data ID whose key-value pairs are used as equality constraints in
2084 the query.
2085 where : `str`, optional
2086 A string expression similar to a SQL WHERE clause. May involve any
2087 column of a dimension table or (as a shortcut for the primary key
2088 column of a dimension table) dimension name. See
2089 :ref:`daf_butler_dimension_expressions` for more information.
2090 bind : `~collections.abc.Mapping`, optional
2091 Mapping containing literal values that should be injected into the
2092 ``where`` expression, keyed by the identifiers they replace. Values
2093 of collection type can be expanded in some cases; see
2094 :ref:`daf_butler_dimension_expressions_identifiers` for more
2095 information.
2096 limit : `int` or `None`, optional
2097 Upper limit on the number of returned records. `None` can be used
2098 if no limit is wanted. A limit of ``0`` means that the query will
2099 be executed and validated but no results will be returned.
2100 If a negative value is given a warning will be issued if the number
2101 of results is capped by that limit. If no limit is provided, by
2102 default a maximum of 20,000 records will be returned.
2103 **kwargs
2104 Additional keyword arguments are forwarded to
2105 `DataCoordinate.standardize` when processing the ``data_id``
2106 argument (and may be used to provide a constraining data ID even
2107 when the ``data_id`` argument is `None`).
2109 Raises
2110 ------
2111 MissingDatasetTypeError
2112 When no dataset types match ``name``, or an explicit (non-glob)
2113 dataset type in ``name`` does not exist.
2114 InvalidQueryError
2115 If the parameters to the query are inconsistent or malformed.
2116 MissingCollectionError
2117 If a given collection is not found.
2119 Returns
2120 -------
2121 refs : `list` [ `DatasetRef` ]
2122 Dataset references matching the given query criteria. Nested data
2123 IDs are guaranteed to include values for all implied dimensions
2124 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2125 include dimension records (`DataCoordinate.hasRecords` will be
2126 `False`).
2127 """
2128 if collections is None:
2129 collections = list(self.collections.defaults)
2130 else:
2131 collections = list(ensure_iterable(collections))
2133 if bind is None:
2134 bind = {}
2135 if data_id is None:
2136 data_id = {}
2138 warn_limit = False
2139 if limit is not None and limit < 0:
2140 # Add one to the limit so we can detect if we have exceeded it.
2141 limit = abs(limit) + 1
2142 warn_limit = True
2144 args = QueryAllDatasetsParameters(
2145 collections=collections,
2146 name=list(ensure_iterable(name)),
2147 find_first=find_first,
2148 data_id=data_id,
2149 where=where,
2150 limit=limit,
2151 bind=bind,
2152 kwargs=kwargs,
2153 with_dimension_records=False,
2154 )
2155 with self._query_all_datasets_by_page(args) as pages:
2156 result = []
2157 for page in pages:
2158 result.extend(page)
2160 if warn_limit and limit is not None and len(result) >= limit:
2161 # Remove the extra dataset we added for the limit check.
2162 result.pop()
2163 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2165 return result
2167 @abstractmethod
2168 def _query_all_datasets_by_page(
2169 self, args: QueryAllDatasetsParameters
2170 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2171 raise NotImplementedError()
2173 def clone(
2174 self,
2175 *,
2176 collections: CollectionArgType | None | EllipsisType = ...,
2177 run: str | None | EllipsisType = ...,
2178 inferDefaults: bool | EllipsisType = ...,
2179 dataId: dict[str, str] | EllipsisType = ...,
2180 metrics: ButlerMetrics | None = None,
2181 ) -> Butler:
2182 """Return a new Butler instance connected to the same repository
2183 as this one, optionally overriding ``collections``, ``run``,
2184 ``inferDefaults``, and default data ID.
2186 Parameters
2187 ----------
2188 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2189 optional
2190 Same as constructor. If omitted, uses value from original object.
2191 run : `str` or `None`, optional
2192 Same as constructor. If `None`, no default run is used. If
2193 omitted, copies value from original object.
2194 inferDefaults : `bool`, optional
2195 Same as constructor. If omitted, copies value from original
2196 object.
2197 dataId : `str`
2198 Same as ``kwargs`` passed to the constructor. If omitted, copies
2199 values from original object.
2200 metrics : `ButlerMetrics` or `None`, optional
2201 Metrics object to record butler statistics.
2202 """
2203 raise NotImplementedError()