Coverage for python / lsst / daf / butler / _butler.py: 30%
310 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-29 01:27 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 metrics : `ButlerMetrics` or `None`
142 External metrics object to be used for tracking butler usage. If `None`
143 a new metrics object is created.
144 **kwargs : `typing.Any`
145 Additional keyword arguments passed to a constructor of actual butler
146 class.
148 Notes
149 -----
150 The preferred way to instantiate Butler is via the `from_config` method.
151 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
152 but ``mypy`` will complain about the former.
153 """
155 def __new__(
156 cls,
157 config: Config | ResourcePathExpression | None = None,
158 *,
159 collections: Any = None,
160 run: str | None = None,
161 searchPaths: Sequence[ResourcePathExpression] | None = None,
162 writeable: bool | None = None,
163 inferDefaults: bool = True,
164 without_datastore: bool = False,
165 metrics: ButlerMetrics | None = None,
166 **kwargs: Any,
167 ) -> Butler:
168 if cls is Butler:
169 return Butler.from_config(
170 config=config,
171 collections=collections,
172 run=run,
173 searchPaths=searchPaths,
174 writeable=writeable,
175 inferDefaults=inferDefaults,
176 without_datastore=without_datastore,
177 metrics=metrics,
178 **kwargs,
179 )
181 # Note: we do not pass any parameters to __new__, Python will pass them
182 # to __init__ after __new__ returns sub-class instance.
183 return super().__new__(cls)
185 @classmethod
186 def from_config(
187 cls,
188 config: Config | ResourcePathExpression | None = None,
189 *,
190 collections: Any = None,
191 run: str | None = None,
192 searchPaths: Sequence[ResourcePathExpression] | None = None,
193 writeable: bool | None = None,
194 inferDefaults: bool = True,
195 without_datastore: bool = False,
196 metrics: ButlerMetrics | None = None,
197 **kwargs: Any,
198 ) -> Butler:
199 """Create butler instance from configuration.
201 Parameters
202 ----------
203 config : `ButlerConfig`, `Config` or `str`, optional
204 Configuration. Anything acceptable to the `ButlerConfig`
205 constructor. If a directory path is given the configuration will be
206 read from a ``butler.yaml`` file in that location. If `None` is
207 given default values will be used. If ``config`` contains "cls" key
208 then its value is used as a name of butler class and it must be a
209 sub-class of this class, otherwise `DirectButler` is instantiated.
210 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
211 An expression specifying the collections to be searched (in order)
212 when reading datasets.
213 This may be a `str` collection name or an iterable thereof.
214 See :ref:`daf_butler_collection_expressions` for more information.
215 These collections are not registered automatically and must be
216 manually registered before they are used by any method, but they
217 may be manually registered after the `Butler` is initialized.
218 run : `str`, optional
219 Name of the `~CollectionType.RUN` collection new datasets should be
220 inserted into. If ``collections`` is `None` and ``run`` is not
221 `None`, ``collections`` will be set to ``[run]``. If not `None`,
222 this collection will automatically be registered. If this is not
223 set (and ``writeable`` is not set either), a read-only butler will
224 be created.
225 searchPaths : `list` of `str`, optional
226 Directory paths to search when calculating the full Butler
227 configuration. Not used if the supplied config is already a
228 `ButlerConfig`.
229 writeable : `bool`, optional
230 Explicitly sets whether the butler supports write operations. If
231 not provided, a read-write butler is created if any of ``run``,
232 ``tags``, or ``chains`` is non-empty.
233 inferDefaults : `bool`, optional
234 If `True` (default) infer default data ID values from the values
235 present in the datasets in ``collections``: if all collections have
236 the same value (or no value) for a governor dimension, that value
237 will be the default for that dimension. Nonexistent collections
238 are ignored. If a default value is provided explicitly for a
239 governor dimension via ``**kwargs``, no default will be inferred
240 for that dimension.
241 without_datastore : `bool`, optional
242 If `True` do not attach a datastore to this butler. Any attempts
243 to use a datastore will fail.
244 metrics : `ButlerMetrics` or `None`, optional
245 Metrics object to record butler usage statistics.
246 **kwargs : `typing.Any`
247 Default data ID key-value pairs. These may only identify
248 "governor" dimensions like ``instrument`` and ``skymap``.
250 Returns
251 -------
252 butler : `Butler`
253 A `Butler` constructed from the given configuration.
255 Notes
256 -----
257 Calling this factory method is identical to calling
258 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
259 complains about ``Butler()`` call.
261 Examples
262 --------
263 While there are many ways to control exactly how a `Butler` interacts
264 with the collections in its `Registry`, the most common cases are still
265 simple.
267 For a read-only `Butler` that searches one collection, do::
269 butler = Butler.from_config(
270 "/path/to/repo", collections=["u/alice/DM-50000"]
271 )
273 For a read-write `Butler` that writes to and reads from a
274 `~CollectionType.RUN` collection::
276 butler = Butler.from_config(
277 "/path/to/repo", run="u/alice/DM-50000/a"
278 )
280 The `Butler` passed to a ``PipelineTask`` is often much more complex,
281 because we want to write to one `~CollectionType.RUN` collection but
282 read from several others (as well)::
284 butler = Butler.from_config(
285 "/path/to/repo",
286 run="u/alice/DM-50000/a",
287 collections=[
288 "u/alice/DM-50000/a",
289 "u/bob/DM-49998",
290 "HSC/defaults",
291 ],
292 )
294 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
295 Datasets will be read first from that run (since it appears first in
296 the chain), and then from ``u/bob/DM-49998`` and finally
297 ``HSC/defaults``.
299 Finally, one can always create a `Butler` with no collections::
301 butler = Butler.from_config("/path/to/repo", writeable=True)
303 This can be extremely useful when you just want to use
304 ``butler.registry``, e.g. for inserting dimension data or managing
305 collections, or when the collections you want to use with the butler
306 are not consistent. Passing ``writeable`` explicitly here is only
307 necessary if you want to be able to make changes to the repo - usually
308 the value for ``writeable`` can be guessed from the collection
309 arguments provided, but it defaults to `False` when there are not
310 collection arguments.
311 """
312 # DirectButler used to have a way to specify a "copy constructor" by
313 # passing the "butler" parameter to its constructor. This has
314 # been moved out of the constructor into Butler.clone().
315 butler = kwargs.pop("butler", None)
316 metrics = metrics if metrics is not None else ButlerMetrics()
317 if butler is not None:
318 if not isinstance(butler, Butler):
319 raise TypeError("'butler' parameter must be a Butler instance")
320 if config is not None or searchPaths is not None or writeable is not None:
321 raise TypeError(
322 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
323 )
324 return butler.clone(
325 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
326 )
328 options = ButlerInstanceOptions(
329 collections=collections,
330 run=run,
331 writeable=writeable,
332 inferDefaults=inferDefaults,
333 metrics=metrics,
334 kwargs=kwargs,
335 )
337 # Load the Butler configuration. This may involve searching the
338 # environment to locate a configuration file.
339 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
340 butler_type = butler_config.get_butler_type()
342 # Make DirectButler if class is not specified.
343 match butler_type:
344 case ButlerType.DIRECT:
345 from .direct_butler import DirectButler
347 return DirectButler.create_from_config(
348 butler_config,
349 options=options,
350 without_datastore=without_datastore,
351 )
352 case ButlerType.REMOTE:
353 from .remote_butler._factory import RemoteButlerFactory
355 # Assume this is being created by a client who would like
356 # default caching of remote datasets.
357 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
358 return factory.create_butler_with_credentials_from_environment(
359 butler_options=options, enable_datastore_cache=True
360 )
361 case _:
362 raise TypeError(f"Unknown Butler type '{butler_type}'")
364 @staticmethod
365 def makeRepo(
366 root: ResourcePathExpression,
367 config: Config | str | None = None,
368 dimensionConfig: Config | str | None = None,
369 standalone: bool = False,
370 searchPaths: list[str] | None = None,
371 forceConfigRoot: bool = True,
372 outfile: ResourcePathExpression | None = None,
373 overwrite: bool = False,
374 ) -> Config:
375 """Create an empty data repository by adding a butler.yaml config
376 to a repository root directory.
378 Parameters
379 ----------
380 root : `lsst.resources.ResourcePathExpression`
381 Path or URI to the root location of the new repository. Will be
382 created if it does not exist.
383 config : `Config` or `str`, optional
384 Configuration to write to the repository, after setting any
385 root-dependent Registry or Datastore config options. Can not
386 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
387 configuration will be used. Root-dependent config options
388 specified in this config are overwritten if ``forceConfigRoot``
389 is `True`.
390 dimensionConfig : `Config` or `str`, optional
391 Configuration for dimensions, will be used to initialize registry
392 database.
393 standalone : `bool`
394 If True, write all expanded defaults, not just customized or
395 repository-specific settings.
396 This (mostly) decouples the repository from the default
397 configuration, insulating it from changes to the defaults (which
398 may be good or bad, depending on the nature of the changes).
399 Future *additions* to the defaults will still be picked up when
400 initializing `Butlers` to repos created with ``standalone=True``.
401 searchPaths : `list` of `str`, optional
402 Directory paths to search when calculating the full butler
403 configuration.
404 forceConfigRoot : `bool`, optional
405 If `False`, any values present in the supplied ``config`` that
406 would normally be reset are not overridden and will appear
407 directly in the output config. This allows non-standard overrides
408 of the root directory for a datastore or registry to be given.
409 If this parameter is `True` the values for ``root`` will be
410 forced into the resulting config if appropriate.
411 outfile : `lss.resources.ResourcePathExpression`, optional
412 If not-`None`, the output configuration will be written to this
413 location rather than into the repository itself. Can be a URI
414 string. Can refer to a directory that will be used to write
415 ``butler.yaml``.
416 overwrite : `bool`, optional
417 Create a new configuration file even if one already exists
418 in the specified output location. Default is to raise
419 an exception.
421 Returns
422 -------
423 config : `Config`
424 The updated `Config` instance written to the repo.
426 Raises
427 ------
428 ValueError
429 Raised if a ButlerConfig or ConfigSubset is passed instead of a
430 regular Config (as these subclasses would make it impossible to
431 support ``standalone=False``).
432 FileExistsError
433 Raised if the output config file already exists.
434 os.error
435 Raised if the directory does not exist, exists but is not a
436 directory, or cannot be created.
438 Notes
439 -----
440 Note that when ``standalone=False`` (the default), the configuration
441 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
442 construct the repository should also be used to construct any Butlers
443 to avoid configuration inconsistencies.
444 """
445 if isinstance(config, ButlerConfig | ConfigSubset):
446 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
448 # Ensure that the root of the repository exists or can be made
449 root_uri = ResourcePath(root, forceDirectory=True)
450 root_uri.mkdir()
452 config = Config(config)
454 # If we are creating a new repo from scratch with relative roots,
455 # do not propagate an explicit root from the config file
456 if "root" in config:
457 del config["root"]
459 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
460 imported_class = doImportType(full["datastore", "cls"])
461 if not issubclass(imported_class, Datastore):
462 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
463 datastoreClass: type[Datastore] = imported_class
464 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
466 # if key exists in given config, parse it, otherwise parse the defaults
467 # in the expanded config
468 if config.get(("registry", "db")):
469 registryConfig = RegistryConfig(config)
470 else:
471 registryConfig = RegistryConfig(full)
472 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
473 if defaultDatabaseUri is not None:
474 Config.updateParameters(
475 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
476 )
477 else:
478 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
480 if standalone:
481 config.merge(full)
482 else:
483 # Always expand the registry.managers section into the per-repo
484 # config, because after the database schema is created, it's not
485 # allowed to change anymore. Note that in the standalone=True
486 # branch, _everything_ in the config is expanded, so there's no
487 # need to special case this.
488 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
489 configURI: ResourcePathExpression
490 if outfile is not None:
491 # When writing to a separate location we must include
492 # the root of the butler repo in the config else it won't know
493 # where to look.
494 config["root"] = root_uri.geturl()
495 configURI = outfile
496 else:
497 configURI = root_uri
498 # Strip obscore configuration, if it is present, before writing config
499 # to a file, obscore config will be stored in registry.
500 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
501 config_to_write = config.copy()
502 del config_to_write[obscore_config_key]
503 config_to_write.dumpToUri(configURI, overwrite=overwrite)
504 # configFile attribute is updated, need to copy it to original.
505 config.configFile = config_to_write.configFile
506 else:
507 config.dumpToUri(configURI, overwrite=overwrite)
509 # Create Registry and populate tables
510 registryConfig = RegistryConfig(config.get("registry"))
511 dimensionConfig = DimensionConfig(dimensionConfig)
512 registry = _RegistryFactory(registryConfig).create_from_config(
513 dimensionConfig=dimensionConfig, butlerRoot=root_uri
514 )
515 registry.close()
517 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
519 return config
521 @classmethod
522 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
523 """Look up the label in a butler repository index.
525 Parameters
526 ----------
527 label : `str`
528 Label of the Butler repository to look up.
529 return_label : `bool`, optional
530 If ``label`` cannot be found in the repository index (either
531 because index is not defined or ``label`` is not in the index) and
532 ``return_label`` is `True` then return ``ResourcePath(label)``.
533 If ``return_label`` is `False` (default) then an exception will be
534 raised instead.
536 Returns
537 -------
538 uri : `lsst.resources.ResourcePath`
539 URI to the Butler repository associated with the given label or
540 default value if it is provided.
542 Raises
543 ------
544 KeyError
545 Raised if the label is not found in the index, or if an index
546 is not defined, and ``return_label`` is `False`.
548 Notes
549 -----
550 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
551 information is discovered.
552 """
553 return ButlerRepoIndex.get_repo_uri(label, return_label)
555 @classmethod
556 def get_known_repos(cls) -> set[str]:
557 """Retrieve the list of known repository labels.
559 Returns
560 -------
561 repos : `set` of `str`
562 All the known labels. Can be empty if no index can be found.
564 Notes
565 -----
566 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
567 information is discovered.
568 """
569 return ButlerRepoIndex.get_known_repos()
571 @classmethod
572 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
573 """Extract the butler label and dataset ID from a dataset URI.
575 Parameters
576 ----------
577 uri : `str`
578 The dataset URI to parse.
580 Returns
581 -------
582 parsed : `ParsedButlerDatasetURI`
583 The label associated with the butler repository from which this
584 dataset originates and the ID of the dataset.
586 Notes
587 -----
588 Supports dataset URIs of the forms
589 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
590 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
591 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
592 can include anything supported by the `Butler` constructor, including
593 paths to repositories and alias labels.
595 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
597 will return a label of ``/repo/main``.
599 This method does not attempt to check that the dataset exists in the
600 labeled butler.
602 Since the IVOID can be issued by any publisher to represent a Butler
603 dataset there is no validation of the path or netloc component of the
604 URI. The only requirement is that there are ``id`` and ``repo`` keys
605 in the ``ivo`` URI query component.
606 """
607 parsed = urllib.parse.urlparse(uri)
608 parsed_scheme = parsed.scheme.lower()
609 if parsed_scheme == "ivo":
610 # Do not validate the netloc or the path values.
611 qs = urllib.parse.parse_qs(parsed.query)
612 if "repo" not in qs or "id" not in qs:
613 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
614 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
615 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
616 label = qs["repo"][0]
617 id_ = qs["id"][0]
618 elif parsed_scheme == "butler":
619 label = parsed.netloc # Butler label is case sensitive.
620 # Need to strip the leading /.
621 id_ = parsed.path[1:]
622 else:
623 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
624 # Strip trailing/leading whitespace from label.
625 label = label.strip()
626 if not label:
627 raise ValueError(f"No butler repository label found in uri {uri!r}")
628 try:
629 dataset_id = uuid.UUID(hex=id_)
630 except Exception as e:
631 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
632 raise
634 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
636 @classmethod
637 def get_dataset_from_uri(
638 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
639 ) -> SpecificButlerDataset:
640 """Get the dataset associated with the given dataset URI.
642 Parameters
643 ----------
644 uri : `str`
645 The URI associated with a dataset.
646 factory : `LabeledButlerFactoryProtocol` or `None`, optional
647 Bound factory function that will be given the butler label
648 and receive a `Butler`. If this is not provided the label
649 will be tried directly.
651 Returns
652 -------
653 result : `SpecificButlerDataset`
654 The butler associated with this URI and the dataset itself.
655 The dataset can be `None` if the UUID is valid but the dataset
656 is not known to this butler.
657 """
658 parsed = cls.parse_dataset_uri(uri)
659 butler: Butler | None = None
660 if factory is not None:
661 # If the label is not recognized, it might be a path.
662 try:
663 butler = factory(parsed.label)
664 except KeyError:
665 pass
666 if butler is None:
667 butler = cls.from_config(parsed.label)
668 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
670 @abstractmethod
671 def _caching_context(self) -> AbstractContextManager[None]:
672 """Context manager that enables caching."""
673 raise NotImplementedError()
675 @abstractmethod
676 def transaction(self) -> AbstractContextManager[None]:
677 """Context manager supporting `Butler` transactions.
679 Transactions can be nested.
680 """
681 raise NotImplementedError()
683 @abstractmethod
684 def put(
685 self,
686 obj: Any,
687 datasetRefOrType: DatasetRef | DatasetType | str,
688 /,
689 dataId: DataId | None = None,
690 *,
691 run: str | None = None,
692 provenance: DatasetProvenance | None = None,
693 **kwargs: Any,
694 ) -> DatasetRef:
695 """Store and register a dataset.
697 Parameters
698 ----------
699 obj : `object`
700 The dataset.
701 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
702 When `DatasetRef` is provided, ``dataId`` should be `None`.
703 Otherwise the `DatasetType` or name thereof. If a fully resolved
704 `DatasetRef` is given the run and ID are used directly.
705 dataId : `dict` or `DataCoordinate`
706 A `dict` of `Dimension` link name, value pairs that label the
707 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
708 should be provided as the second argument.
709 run : `str`, optional
710 The name of the run the dataset should be added to, overriding
711 ``self.run``. Not used if a resolved `DatasetRef` is provided.
712 provenance : `DatasetProvenance` or `None`, optional
713 Any provenance that should be attached to the serialized dataset.
714 Not supported by all serialization mechanisms.
715 **kwargs
716 Additional keyword arguments used to augment or construct a
717 `DataCoordinate`. See `DataCoordinate.standardize`
718 parameters. Not used if a resolve `DatasetRef` is provided.
720 Returns
721 -------
722 ref : `DatasetRef`
723 A reference to the stored dataset, updated with the correct id if
724 given.
726 Raises
727 ------
728 TypeError
729 Raised if the butler is read-only or if no run has been provided.
730 """
731 raise NotImplementedError()
733 @abstractmethod
734 def getDeferred(
735 self,
736 datasetRefOrType: DatasetRef | DatasetType | str,
737 /,
738 dataId: DataId | None = None,
739 *,
740 parameters: dict | None = None,
741 collections: Any = None,
742 storageClass: str | StorageClass | None = None,
743 timespan: Timespan | None = None,
744 **kwargs: Any,
745 ) -> DeferredDatasetHandle:
746 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
747 after an immediate registry lookup.
749 Parameters
750 ----------
751 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
752 When `DatasetRef` the `dataId` should be `None`.
753 Otherwise the `DatasetType` or name thereof.
754 dataId : `dict` or `DataCoordinate`, optional
755 A `dict` of `Dimension` link name, value pairs that label the
756 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
757 should be provided as the first argument.
758 parameters : `dict`
759 Additional StorageClass-defined options to control reading,
760 typically used to efficiently read only a subset of the dataset.
761 collections : Any, optional
762 Collections to be searched, overriding ``self.collections``.
763 Can be any of the types supported by the ``collections`` argument
764 to butler construction.
765 storageClass : `StorageClass` or `str`, optional
766 The storage class to be used to override the Python type
767 returned by this method. By default the returned type matches
768 the dataset type definition for this dataset. Specifying a
769 read `StorageClass` can force a different type to be returned.
770 This type must be compatible with the original type.
771 timespan : `Timespan` or `None`, optional
772 A timespan that the validity range of the dataset must overlap.
773 If not provided and this is a calibration dataset type, an attempt
774 will be made to find the timespan from any temporal coordinate
775 in the data ID.
776 **kwargs
777 Additional keyword arguments used to augment or construct a
778 `DataId`. See `DataId` parameters.
780 Returns
781 -------
782 obj : `DeferredDatasetHandle`
783 A handle which can be used to retrieve a dataset at a later time.
785 Raises
786 ------
787 LookupError
788 Raised if no matching dataset exists in the `Registry` or
789 datastore.
790 ValueError
791 Raised if a resolved `DatasetRef` was passed as an input, but it
792 differs from the one found in the registry.
793 TypeError
794 Raised if no collections were provided.
795 """
796 raise NotImplementedError()
798 @abstractmethod
799 def get(
800 self,
801 datasetRefOrType: DatasetRef | DatasetType | str,
802 /,
803 dataId: DataId | None = None,
804 *,
805 parameters: dict[str, Any] | None = None,
806 collections: Any = None,
807 storageClass: StorageClass | str | None = None,
808 timespan: Timespan | None = None,
809 **kwargs: Any,
810 ) -> Any:
811 """Retrieve a stored dataset.
813 Parameters
814 ----------
815 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
816 When `DatasetRef` the `dataId` should be `None`.
817 Otherwise the `DatasetType` or name thereof.
818 If a resolved `DatasetRef`, the associated dataset
819 is returned directly without additional querying.
820 dataId : `dict` or `DataCoordinate`
821 A `dict` of `Dimension` link name, value pairs that label the
822 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
823 should be provided as the first argument.
824 parameters : `dict`
825 Additional StorageClass-defined options to control reading,
826 typically used to efficiently read only a subset of the dataset.
827 collections : Any, optional
828 Collections to be searched, overriding ``self.collections``.
829 Can be any of the types supported by the ``collections`` argument
830 to butler construction.
831 storageClass : `StorageClass` or `str`, optional
832 The storage class to be used to override the Python type
833 returned by this method. By default the returned type matches
834 the dataset type definition for this dataset. Specifying a
835 read `StorageClass` can force a different type to be returned.
836 This type must be compatible with the original type.
837 timespan : `Timespan` or `None`, optional
838 A timespan that the validity range of the dataset must overlap.
839 If not provided and this is a calibration dataset type, an attempt
840 will be made to find the timespan from any temporal coordinate
841 in the data ID.
842 **kwargs
843 Additional keyword arguments used to augment or construct a
844 `DataCoordinate`. See `DataCoordinate.standardize`
845 parameters.
847 Returns
848 -------
849 obj : `object`
850 The dataset.
852 Raises
853 ------
854 LookupError
855 Raised if no matching dataset exists in the `Registry`.
856 TypeError
857 Raised if no collections were provided.
859 Notes
860 -----
861 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
862 this method requires that the given data ID include temporal dimensions
863 beyond the dimensions of the dataset type itself, in order to find the
864 dataset with the appropriate validity range. For example, a "bias"
865 dataset with native dimensions ``{instrument, detector}`` could be
866 fetched with a ``{instrument, detector, exposure}`` data ID, because
867 ``exposure`` is a temporal dimension.
868 """
869 raise NotImplementedError()
871 @abstractmethod
872 def getURIs(
873 self,
874 datasetRefOrType: DatasetRef | DatasetType | str,
875 /,
876 dataId: DataId | None = None,
877 *,
878 predict: bool = False,
879 collections: Any = None,
880 run: str | None = None,
881 **kwargs: Any,
882 ) -> DatasetRefURIs:
883 """Return the URIs associated with the dataset.
885 Parameters
886 ----------
887 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
888 When `DatasetRef` the `dataId` should be `None`.
889 Otherwise the `DatasetType` or name thereof.
890 dataId : `dict` or `DataCoordinate`
891 A `dict` of `Dimension` link name, value pairs that label the
892 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
893 should be provided as the first argument.
894 predict : `bool`
895 If `True`, allow URIs to be returned of datasets that have not
896 been written.
897 collections : Any, optional
898 Collections to be searched, overriding ``self.collections``.
899 Can be any of the types supported by the ``collections`` argument
900 to butler construction.
901 run : `str`, optional
902 Run to use for predictions, overriding ``self.run``.
903 **kwargs
904 Additional keyword arguments used to augment or construct a
905 `DataCoordinate`. See `DataCoordinate.standardize`
906 parameters.
908 Returns
909 -------
910 uris : `DatasetRefURIs`
911 The URI to the primary artifact associated with this dataset (if
912 the dataset was disassembled within the datastore this may be
913 `None`), and the URIs to any components associated with the dataset
914 artifact. (can be empty if there are no components).
915 """
916 raise NotImplementedError()
918 def getURI(
919 self,
920 datasetRefOrType: DatasetRef | DatasetType | str,
921 /,
922 dataId: DataId | None = None,
923 *,
924 predict: bool = False,
925 collections: Any = None,
926 run: str | None = None,
927 **kwargs: Any,
928 ) -> ResourcePath:
929 """Return the URI to the Dataset.
931 Parameters
932 ----------
933 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
934 When `DatasetRef` the `dataId` should be `None`.
935 Otherwise the `DatasetType` or name thereof.
936 dataId : `dict` or `DataCoordinate`
937 A `dict` of `Dimension` link name, value pairs that label the
938 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
939 should be provided as the first argument.
940 predict : `bool`
941 If `True`, allow URIs to be returned of datasets that have not
942 been written.
943 collections : Any, optional
944 Collections to be searched, overriding ``self.collections``.
945 Can be any of the types supported by the ``collections`` argument
946 to butler construction.
947 run : `str`, optional
948 Run to use for predictions, overriding ``self.run``.
949 **kwargs
950 Additional keyword arguments used to augment or construct a
951 `DataCoordinate`. See `DataCoordinate.standardize`
952 parameters.
954 Returns
955 -------
956 uri : `lsst.resources.ResourcePath`
957 URI pointing to the Dataset within the datastore. If the
958 Dataset does not exist in the datastore, and if ``predict`` is
959 `True`, the URI will be a prediction and will include a URI
960 fragment "#predicted".
961 If the datastore does not have entities that relate well
962 to the concept of a URI the returned URI string will be
963 descriptive. The returned URI is not guaranteed to be obtainable.
965 Raises
966 ------
967 LookupError
968 A URI has been requested for a dataset that does not exist and
969 guessing is not allowed.
970 ValueError
971 Raised if a resolved `DatasetRef` was passed as an input, but it
972 differs from the one found in the registry.
973 TypeError
974 Raised if no collections were provided.
975 RuntimeError
976 Raised if a URI is requested for a dataset that consists of
977 multiple artifacts.
978 """
979 primary, components = self.getURIs(
980 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
981 )
983 if primary is None or components:
984 raise RuntimeError(
985 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
986 "Use Butler.getURIs() instead."
987 )
988 return primary
990 @abstractmethod
991 def get_dataset_type(self, name: str) -> DatasetType:
992 """Get the `DatasetType`.
994 Parameters
995 ----------
996 name : `str`
997 Name of the type.
999 Returns
1000 -------
1001 type : `DatasetType`
1002 The `DatasetType` associated with the given name.
1004 Raises
1005 ------
1006 lsst.daf.butler.MissingDatasetTypeError
1007 Raised if the requested dataset type has not been registered.
1009 Notes
1010 -----
1011 This method handles component dataset types automatically, though most
1012 other operations do not.
1013 """
1014 raise NotImplementedError()
1016 @abstractmethod
1017 def get_dataset(
1018 self,
1019 id: DatasetId | str,
1020 *,
1021 storage_class: str | StorageClass | None = None,
1022 dimension_records: bool = False,
1023 datastore_records: bool = False,
1024 ) -> DatasetRef | None:
1025 """Retrieve a Dataset entry.
1027 Parameters
1028 ----------
1029 id : `DatasetId`
1030 The unique identifier for the dataset, as an instance of
1031 `uuid.UUID` or a string containing a hexadecimal number.
1032 storage_class : `str` or `StorageClass` or `None`
1033 A storage class to use when creating the returned entry. If given
1034 it must be compatible with the default storage class.
1035 dimension_records : `bool`, optional
1036 If `True` the ref will be expanded and contain dimension records.
1037 datastore_records : `bool`, optional
1038 If `True` the ref will contain associated datastore records.
1040 Returns
1041 -------
1042 ref : `DatasetRef` or `None`
1043 A ref to the Dataset, or `None` if no matching Dataset
1044 was found.
1045 """
1046 raise NotImplementedError()
1048 @abstractmethod
1049 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1050 """Retrieve a list of dataset entries.
1052 Parameters
1053 ----------
1054 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1055 The unique identifiers for the datasets, as instances of
1056 `uuid.UUID` or strings containing a hexadecimal number.
1058 Returns
1059 -------
1060 refs : `list` [ `DatasetRef` ]
1061 A list containing a `DatasetRef` for each of the given dataset IDs.
1062 If a dataset was not found, no error is thrown -- it is just not
1063 included in the list. The returned datasets are in no particular
1064 order.
1065 """
1066 raise NotImplementedError()
1068 @abstractmethod
1069 def find_dataset(
1070 self,
1071 dataset_type: DatasetType | str,
1072 data_id: DataId | None = None,
1073 *,
1074 collections: str | Sequence[str] | None = None,
1075 timespan: Timespan | None = None,
1076 storage_class: str | StorageClass | None = None,
1077 dimension_records: bool = False,
1078 datastore_records: bool = False,
1079 **kwargs: Any,
1080 ) -> DatasetRef | None:
1081 """Find a dataset given its `DatasetType` and data ID.
1083 This can be used to obtain a `DatasetRef` that permits the dataset to
1084 be read from a `Datastore`. If the dataset is a component and can not
1085 be found using the provided dataset type, a dataset ref for the parent
1086 will be returned instead but with the correct dataset type.
1088 Parameters
1089 ----------
1090 dataset_type : `DatasetType` or `str`
1091 A `DatasetType` or the name of one. If this is a `DatasetType`
1092 instance, its storage class will be respected and propagated to
1093 the output, even if it differs from the dataset type definition
1094 in the registry, as long as the storage classes are convertible.
1095 data_id : `dict` or `DataCoordinate`, optional
1096 A `dict`-like object containing the `Dimension` links that identify
1097 the dataset within a collection. If it is a `dict` the dataId
1098 can include dimension record values such as ``day_obs`` and
1099 ``seq_num`` or ``full_name`` that can be used to derive the
1100 primary dimension.
1101 collections : `str` or `list` [`str`], optional
1102 A an ordered list of collections to search for the dataset.
1103 Defaults to ``self.defaults.collections``.
1104 timespan : `Timespan`, optional
1105 A timespan that the validity range of the dataset must overlap.
1106 If not provided, any `~CollectionType.CALIBRATION` collections
1107 matched by the ``collections`` argument will not be searched.
1108 storage_class : `str` or `StorageClass` or `None`
1109 A storage class to use when creating the returned entry. If given
1110 it must be compatible with the default storage class.
1111 dimension_records : `bool`, optional
1112 If `True` the ref will be expanded and contain dimension records.
1113 datastore_records : `bool`, optional
1114 If `True` the ref will contain associated datastore records.
1115 **kwargs
1116 Additional keyword arguments passed to
1117 `DataCoordinate.standardize` to convert ``dataId`` to a true
1118 `DataCoordinate` or augment an existing one. This can also include
1119 dimension record metadata that can be used to derive a primary
1120 dimension value.
1122 Returns
1123 -------
1124 ref : `DatasetRef`
1125 A reference to the dataset, or `None` if no matching Dataset
1126 was found.
1128 Raises
1129 ------
1130 lsst.daf.butler.NoDefaultCollectionError
1131 Raised if ``collections`` is `None` and
1132 ``self.collections`` is `None`.
1133 LookupError
1134 Raised if one or more data ID keys are missing.
1135 lsst.daf.butler.MissingDatasetTypeError
1136 Raised if the dataset type does not exist.
1137 lsst.daf.butler.MissingCollectionError
1138 Raised if any of ``collections`` does not exist in the registry.
1140 Notes
1141 -----
1142 This method simply returns `None` and does not raise an exception even
1143 when the set of collections searched is intrinsically incompatible with
1144 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1145 only `~CollectionType.CALIBRATION` collections are being searched.
1146 This may make it harder to debug some lookup failures, but the behavior
1147 is intentional; we consider it more important that failed searches are
1148 reported consistently, regardless of the reason, and that adding
1149 additional collections that do not contain a match to the search path
1150 never changes the behavior.
1152 This method handles component dataset types automatically, though most
1153 other query operations do not.
1154 """
1155 raise NotImplementedError()
1157 @abstractmethod
1158 def retrieve_artifacts_zip(
1159 self,
1160 refs: Iterable[DatasetRef],
1161 destination: ResourcePathExpression,
1162 overwrite: bool = True,
1163 ) -> ResourcePath:
1164 """Retrieve artifacts from a Butler and place in ZIP file.
1166 Parameters
1167 ----------
1168 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1169 The datasets to be included in the zip file.
1170 destination : `lsst.resources.ResourcePathExpression`
1171 Directory to write the new ZIP file. This directory will
1172 also be used as a staging area for the datasets being downloaded
1173 from the datastore.
1174 overwrite : `bool`, optional
1175 If `False` the output Zip will not be written if a file of the
1176 same name is already present in ``destination``.
1178 Returns
1179 -------
1180 zip_file : `lsst.resources.ResourcePath`
1181 The path to the new ZIP file.
1183 Raises
1184 ------
1185 ValueError
1186 Raised if there are no refs to retrieve.
1187 """
1188 raise NotImplementedError()
1190 @abstractmethod
1191 def retrieveArtifacts(
1192 self,
1193 refs: Iterable[DatasetRef],
1194 destination: ResourcePathExpression,
1195 transfer: str = "auto",
1196 preserve_path: bool = True,
1197 overwrite: bool = False,
1198 ) -> list[ResourcePath]:
1199 """Retrieve the artifacts associated with the supplied refs.
1201 Parameters
1202 ----------
1203 refs : iterable of `DatasetRef`
1204 The datasets for which artifacts are to be retrieved.
1205 A single ref can result in multiple artifacts. The refs must
1206 be resolved.
1207 destination : `lsst.resources.ResourcePath` or `str`
1208 Location to write the artifacts.
1209 transfer : `str`, optional
1210 Method to use to transfer the artifacts. Must be one of the options
1211 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1212 "move" is not allowed.
1213 preserve_path : `bool`, optional
1214 If `True` the full path of the artifact within the datastore
1215 is preserved. If `False` the final file component of the path
1216 is used.
1217 overwrite : `bool`, optional
1218 If `True` allow transfers to overwrite existing files at the
1219 destination.
1221 Returns
1222 -------
1223 targets : `list` of `lsst.resources.ResourcePath`
1224 URIs of file artifacts in destination location. Order is not
1225 preserved.
1227 Notes
1228 -----
1229 For non-file datastores the artifacts written to the destination
1230 may not match the representation inside the datastore. For example
1231 a hierarchical data structure in a NoSQL database may well be stored
1232 as a JSON file.
1233 """
1234 raise NotImplementedError()
1236 @abstractmethod
1237 def exists(
1238 self,
1239 dataset_ref_or_type: DatasetRef | DatasetType | str,
1240 /,
1241 data_id: DataId | None = None,
1242 *,
1243 full_check: bool = True,
1244 collections: Any = None,
1245 **kwargs: Any,
1246 ) -> DatasetExistence:
1247 """Indicate whether a dataset is known to Butler registry and
1248 datastore.
1250 Parameters
1251 ----------
1252 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1253 When `DatasetRef` the `dataId` should be `None`.
1254 Otherwise the `DatasetType` or name thereof.
1255 data_id : `dict` or `DataCoordinate`
1256 A `dict` of `Dimension` link name, value pairs that label the
1257 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1258 should be provided as the first argument.
1259 full_check : `bool`, optional
1260 If `True`, a check will be made for the actual existence of a
1261 dataset artifact. This will involve additional overhead due to
1262 the need to query an external system. If `False`, this check will
1263 be omitted, and the registry and datastore will solely be asked
1264 if they know about the dataset but no direct check for the
1265 artifact will be performed.
1266 collections : Any, optional
1267 Collections to be searched, overriding ``self.collections``.
1268 Can be any of the types supported by the ``collections`` argument
1269 to butler construction.
1270 **kwargs
1271 Additional keyword arguments used to augment or construct a
1272 `DataCoordinate`. See `DataCoordinate.standardize`
1273 parameters.
1275 Returns
1276 -------
1277 existence : `DatasetExistence`
1278 Object indicating whether the dataset is known to registry and
1279 datastore. Evaluates to `True` if the dataset is present and known
1280 to both.
1281 """
1282 raise NotImplementedError()
1284 @abstractmethod
1285 def _exists_many(
1286 self,
1287 refs: Iterable[DatasetRef],
1288 /,
1289 *,
1290 full_check: bool = True,
1291 ) -> dict[DatasetRef, DatasetExistence]:
1292 """Indicate whether multiple datasets are known to Butler registry and
1293 datastore.
1295 This is an experimental API that may change at any moment.
1297 Parameters
1298 ----------
1299 refs : iterable of `DatasetRef`
1300 The datasets to be checked.
1301 full_check : `bool`, optional
1302 If `True`, a check will be made for the actual existence of each
1303 dataset artifact. This will involve additional overhead due to
1304 the need to query an external system. If `False`, this check will
1305 be omitted, and the registry and datastore will solely be asked
1306 if they know about the dataset(s) but no direct check for the
1307 artifact(s) will be performed.
1309 Returns
1310 -------
1311 existence : dict of [`DatasetRef`, `DatasetExistence`]
1312 Mapping from the given dataset refs to an enum indicating the
1313 status of the dataset in registry and datastore.
1314 Each value evaluates to `True` if the dataset is present and known
1315 to both.
1316 """
1317 raise NotImplementedError()
1319 @abstractmethod
1320 def removeRuns(
1321 self,
1322 names: Iterable[str],
1323 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1324 *,
1325 unlink_from_chains: bool = False,
1326 ) -> None:
1327 """Remove one or more `~CollectionType.RUN` collections and the
1328 datasets within them.
1330 Parameters
1331 ----------
1332 names : `~collections.abc.Iterable` [ `str` ]
1333 The names of the collections to remove.
1334 unstore : `bool`, optional
1335 If `True` (default), delete datasets from all datastores in which
1336 they are present, and attempt to rollback the registry deletions if
1337 datastore deletions fail (which may not always be possible). If
1338 `False`, datastore records for these datasets are still removed,
1339 but any artifacts (e.g. files) will not be. This parameter is now
1340 deprecated and no longer has any effect. Files are always deleted
1341 from datastores unless they were ingested using full URIs.
1342 unlink_from_chains : `bool`, optional
1343 If `True` remove the RUN collection from any chains prior to
1344 removing the RUN. If `False` the removal will fail if any chains
1345 still refer to the RUN.
1347 Raises
1348 ------
1349 TypeError
1350 Raised if one or more collections are not of type
1351 `~CollectionType.RUN`.
1352 """
1353 raise NotImplementedError()
1355 @abstractmethod
1356 def ingest(
1357 self,
1358 *datasets: FileDataset,
1359 transfer: str | None = "auto",
1360 record_validation_info: bool = True,
1361 skip_existing: bool = False,
1362 ) -> None:
1363 """Store and register one or more datasets that already exist on disk.
1365 Parameters
1366 ----------
1367 *datasets : `FileDataset`
1368 Each positional argument is a struct containing information about
1369 a file to be ingested, including its URI (either absolute or
1370 relative to the datastore root, if applicable), a resolved
1371 `DatasetRef`, and optionally a formatter class or its
1372 fully-qualified string name. If a formatter is not provided, the
1373 formatter that would be used for `put` is assumed. On successful
1374 ingest all `FileDataset.formatter` attributes will be set to the
1375 formatter class used. `FileDataset.path` attributes may be modified
1376 to put paths in whatever the datastore considers a standardized
1377 form.
1378 transfer : `str`, optional
1379 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1380 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1381 transfer the file.
1382 record_validation_info : `bool`, optional
1383 If `True`, the default, the datastore can record validation
1384 information associated with the file. If `False` the datastore
1385 will not attempt to track any information such as checksums
1386 or file sizes. This can be useful if such information is tracked
1387 in an external system or if the file is to be compressed in place.
1388 It is up to the datastore whether this parameter is relevant.
1389 skip_existing : `bool`, optional
1390 If `True`, a dataset will not be ingested if a dataset with the
1391 same dataset ID already exists in the datastore.
1392 If `False` (the default), a `ConflictingDefinitionError` will be
1393 raised if any datasets with the same dataset ID already exist
1394 in the datastore.
1396 Returns
1397 -------
1398 None
1400 Raises
1401 ------
1402 TypeError
1403 Raised if the butler is read-only or if no run was provided.
1404 NotImplementedError
1405 Raised if the `Datastore` does not support the given transfer mode.
1406 DatasetTypeNotSupportedError
1407 Raised if one or more files to be ingested have a dataset type that
1408 is not supported by the `Datastore`..
1409 FileNotFoundError
1410 Raised if one of the given files does not exist.
1411 FileExistsError
1412 Raised if transfer is not `None` but the (internal) location the
1413 file would be moved to is already occupied.
1414 ConflictingDefinitionError
1415 Raised if a dataset already exists in the repository and
1416 ``skip_existing`` is `False`.
1418 Notes
1419 -----
1420 This operation is not fully exception safe: if a database operation
1421 fails, the given `FileDataset` instances may be only partially updated.
1423 It is atomic in terms of database operations (they will either all
1424 succeed or all fail) providing the database engine implements
1425 transactions correctly. It will attempt to be atomic in terms of
1426 filesystem operations as well, but this cannot be implemented
1427 rigorously for most datastores.
1428 """
1429 raise NotImplementedError()
1431 @abstractmethod
1432 def ingest_zip(
1433 self,
1434 zip_file: ResourcePathExpression,
1435 transfer: str = "auto",
1436 *,
1437 transfer_dimensions: bool = False,
1438 dry_run: bool = False,
1439 ) -> None:
1440 """Ingest a Zip file into this butler.
1442 The Zip file must have been created by `retrieve_artifacts_zip`.
1444 Parameters
1445 ----------
1446 zip_file : `lsst.resources.ResourcePathExpression`
1447 Path to the Zip file.
1448 transfer : `str`, optional
1449 Method to use to transfer the Zip into the datastore.
1450 transfer_dimensions : `bool`, optional
1451 If `True`, dimension record data associated with the new datasets
1452 will be transferred from the Zip file, if present.
1453 dry_run : `bool`, optional
1454 If `True` the ingest will be processed without any modifications
1455 made to the target butler and as if the target butler did not
1456 have any of the datasets.
1458 Notes
1459 -----
1460 Run collections and dataset types are created as needed.
1461 """
1462 raise NotImplementedError()
1464 @abstractmethod
1465 def export(
1466 self,
1467 *,
1468 directory: str | None = None,
1469 filename: str | None = None,
1470 format: str | None = None,
1471 transfer: str | None = None,
1472 ) -> AbstractContextManager[RepoExportContext]:
1473 """Export datasets from the repository represented by this `Butler`.
1475 This method is a context manager that returns a helper object
1476 (`RepoExportContext`) that is used to indicate what information from
1477 the repository should be exported.
1479 Parameters
1480 ----------
1481 directory : `str`, optional
1482 Directory dataset files should be written to if ``transfer`` is not
1483 `None`.
1484 filename : `str`, optional
1485 Name for the file that will include database information associated
1486 with the exported datasets. If this is not an absolute path and
1487 ``directory`` is not `None`, it will be written to ``directory``
1488 instead of the current working directory. Defaults to
1489 "export.{format}".
1490 format : `str`, optional
1491 File format for the database information file. If `None`, the
1492 extension of ``filename`` will be used.
1493 transfer : `str`, optional
1494 Transfer mode passed to `Datastore.export`.
1496 Raises
1497 ------
1498 TypeError
1499 Raised if the set of arguments passed is inconsistent.
1501 Examples
1502 --------
1503 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1504 methods are used to provide the iterables over data IDs and/or datasets
1505 to be exported::
1507 with butler.export("exports.yaml") as export:
1508 # Export all flats, but none of the dimension element rows
1509 # (i.e. data ID information) associated with them.
1510 export.saveDatasets(
1511 butler.registry.queryDatasets("flat"), elements=()
1512 )
1513 # Export all datasets that start with "deepCoadd_" and all of
1514 # their associated data ID information.
1515 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1516 """
1517 raise NotImplementedError()
1519 @abstractmethod
1520 def import_(
1521 self,
1522 *,
1523 directory: ResourcePathExpression | None = None,
1524 filename: ResourcePathExpression | TextIO | None = None,
1525 format: str | None = None,
1526 transfer: str | None = None,
1527 skip_dimensions: set | None = None,
1528 record_validation_info: bool = True,
1529 without_datastore: bool = False,
1530 ) -> None:
1531 """Import datasets into this repository that were exported from a
1532 different butler repository via `~lsst.daf.butler.Butler.export`.
1534 Parameters
1535 ----------
1536 directory : `~lsst.resources.ResourcePathExpression`, optional
1537 Directory containing dataset files to import from. If `None`,
1538 ``filename`` and all dataset file paths specified therein must
1539 be absolute.
1540 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1541 A stream or name of file that contains database information
1542 associated with the exported datasets, typically generated by
1543 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1544 `~lsst.resources.ResourcePath` and is not an absolute path,
1545 it will first be looked for relative to ``directory`` and if not
1546 found there it will be looked for in the current working
1547 directory. Defaults to "export.{format}".
1548 format : `str`, optional
1549 File format for ``filename``. If `None`, the extension of
1550 ``filename`` will be used.
1551 transfer : `str`, optional
1552 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1553 skip_dimensions : `set`, optional
1554 Names of dimensions that should be skipped and not imported.
1555 record_validation_info : `bool`, optional
1556 If `True`, the default, the datastore can record validation
1557 information associated with the file. If `False` the datastore
1558 will not attempt to track any information such as checksums
1559 or file sizes. This can be useful if such information is tracked
1560 in an external system or if the file is to be compressed in place.
1561 It is up to the datastore whether this parameter is relevant.
1562 without_datastore : `bool`, optional
1563 If `True` only registry records will be imported and the datastore
1564 will be ignored.
1566 Raises
1567 ------
1568 TypeError
1569 Raised if the set of arguments passed is inconsistent, or if the
1570 butler is read-only.
1571 """
1572 raise NotImplementedError()
1574 @abstractmethod
1575 def transfer_dimension_records_from(
1576 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef | DataCoordinate]
1577 ) -> None:
1578 """Transfer dimension records to this Butler from another Butler.
1580 Parameters
1581 ----------
1582 source_butler : `LimitedButler` or `Butler`
1583 Butler from which the records are to be transferred. If data IDs
1584 in ``source_refs`` are not expanded then this has to be a full
1585 `Butler` whose registry will be used to expand data IDs. If the
1586 source refs contain coordinates that are used to populate other
1587 records then this will also need to be a full `Butler`.
1588 source_refs : iterable of `DatasetRef` or `DataCoordinate`
1589 Datasets or data IDs defined in the source butler whose dimension
1590 records should be transferred to this butler.
1591 """
1592 raise NotImplementedError()
1594 @abstractmethod
1595 def transfer_from(
1596 self,
1597 source_butler: LimitedButler,
1598 source_refs: Iterable[DatasetRef],
1599 transfer: str = "auto",
1600 skip_missing: bool = True,
1601 register_dataset_types: bool = False,
1602 transfer_dimensions: bool = False,
1603 dry_run: bool = False,
1604 ) -> Collection[DatasetRef]:
1605 """Transfer datasets to this Butler from a run in another Butler.
1607 Parameters
1608 ----------
1609 source_butler : `LimitedButler`
1610 Butler from which the datasets are to be transferred. If data IDs
1611 in ``source_refs`` are not expanded then this has to be a full
1612 `Butler` whose registry will be used to expand data IDs.
1613 source_refs : iterable of `DatasetRef`
1614 Datasets defined in the source butler that should be transferred to
1615 this butler. In most circumstances, ``transfer_from`` is faster if
1616 the dataset refs are expanded.
1617 transfer : `str`, optional
1618 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1619 skip_missing : `bool`
1620 If `True`, datasets with no datastore artifact associated with
1621 them are not transferred. If `False` a registry entry will be
1622 created even if no datastore record is created (and so will
1623 look equivalent to the dataset being unstored).
1624 register_dataset_types : `bool`
1625 If `True` any missing dataset types are registered. Otherwise
1626 an exception is raised.
1627 transfer_dimensions : `bool`, optional
1628 If `True`, dimension record data associated with the new datasets
1629 will be transferred.
1630 dry_run : `bool`, optional
1631 If `True` the transfer will be processed without any modifications
1632 made to the target butler and as if the target butler did not
1633 have any of the datasets.
1635 Returns
1636 -------
1637 refs : `list` of `DatasetRef`
1638 The refs added to this Butler.
1640 Notes
1641 -----
1642 The datastore artifact has to exist for a transfer
1643 to be made but non-existence is not an error.
1645 Datasets that already exist in this run will be skipped.
1647 The datasets are imported as part of a transaction, although
1648 dataset types are registered before the transaction is started.
1649 This means that it is possible for a dataset type to be registered
1650 even though transfer has failed.
1651 """
1652 raise NotImplementedError()
1654 @abstractmethod
1655 def validateConfiguration(
1656 self,
1657 logFailures: bool = False,
1658 datasetTypeNames: Iterable[str] | None = None,
1659 ignore: Iterable[str] | None = None,
1660 ) -> None:
1661 """Validate butler configuration.
1663 Checks that each `DatasetType` can be stored in the `Datastore`.
1665 Parameters
1666 ----------
1667 logFailures : `bool`, optional
1668 If `True`, output a log message for every validation error
1669 detected.
1670 datasetTypeNames : iterable of `str`, optional
1671 The `DatasetType` names that should be checked. This allows
1672 only a subset to be selected.
1673 ignore : iterable of `str`, optional
1674 Names of DatasetTypes to skip over. This can be used to skip
1675 known problems. If a named `DatasetType` corresponds to a
1676 composite, all components of that `DatasetType` will also be
1677 ignored.
1679 Raises
1680 ------
1681 ButlerValidationError
1682 Raised if there is some inconsistency with how this Butler
1683 is configured.
1684 """
1685 raise NotImplementedError()
1687 @property
1688 @abstractmethod
1689 def collection_chains(self) -> ButlerCollections:
1690 """Object with methods for modifying collection chains
1691 (`~lsst.daf.butler.ButlerCollections`).
1693 Deprecated. Replaced with ``collections`` property.
1694 """
1695 raise NotImplementedError()
1697 @property
1698 @abstractmethod
1699 def collections(self) -> ButlerCollections:
1700 """Object with methods for modifying and querying collections
1701 (`~lsst.daf.butler.ButlerCollections`).
1703 Use of this object is preferred over `registry` wherever possible.
1704 """
1705 raise NotImplementedError()
1707 @property
1708 @abstractmethod
1709 def run(self) -> str | None:
1710 """Name of the run this butler writes outputs to by default (`str` or
1711 `None`).
1712 """
1713 raise NotImplementedError()
1715 @property
1716 @abstractmethod
1717 def registry(self) -> Registry:
1718 """The object that manages dataset metadata and relationships
1719 (`Registry`).
1721 Many operations that don't involve reading or writing butler datasets
1722 are accessible only via `Registry` methods. Eventually these methods
1723 will be replaced by equivalent `Butler` methods.
1724 """
1725 raise NotImplementedError()
1727 @abstractmethod
1728 def query(self) -> AbstractContextManager[Query]:
1729 """Context manager returning a `Query` object used for construction
1730 and execution of complex queries.
1731 """
1732 raise NotImplementedError()
1734 def query_data_ids(
1735 self,
1736 dimensions: DimensionGroup | Iterable[str] | str,
1737 *,
1738 data_id: DataId | None = None,
1739 where: str = "",
1740 bind: Mapping[str, Any] | None = None,
1741 with_dimension_records: bool = False,
1742 order_by: Iterable[str] | str | None = None,
1743 limit: int | None = -20_000,
1744 explain: bool = True,
1745 **kwargs: Any,
1746 ) -> list[DataCoordinate]:
1747 """Query for data IDs matching user-provided criteria.
1749 Parameters
1750 ----------
1751 dimensions : `DimensionGroup`, `str`, or \
1752 `~collections.abc.Iterable` [`str`]
1753 The dimensions of the data IDs to yield, as either `DimensionGroup`
1754 instances or `str`. Will be automatically expanded to a complete
1755 `DimensionGroup`.
1756 data_id : `dict` or `DataCoordinate`, optional
1757 A data ID whose key-value pairs are used as equality constraints
1758 in the query.
1759 where : `str`, optional
1760 A string expression similar to a SQL WHERE clause. May involve
1761 any column of a dimension table or (as a shortcut for the primary
1762 key column of a dimension table) dimension name. See
1763 :ref:`daf_butler_dimension_expressions` for more information.
1764 bind : `~collections.abc.Mapping`, optional
1765 Mapping containing literal values that should be injected into the
1766 ``where`` expression, keyed by the identifiers they replace.
1767 Values of collection type can be expanded in some cases; see
1768 :ref:`daf_butler_dimension_expressions_identifiers` for more
1769 information.
1770 with_dimension_records : `bool`, optional
1771 If `True` (default is `False`) then returned data IDs will have
1772 dimension records.
1773 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1774 Names of the columns/dimensions to use for ordering returned data
1775 IDs. Column name can be prefixed with minus (``-``) to use
1776 descending ordering.
1777 limit : `int` or `None`, optional
1778 Upper limit on the number of returned records. `None` can be used
1779 if no limit is wanted. A limit of ``0`` means that the query will
1780 be executed and validated but no results will be returned. In this
1781 case there will be no exception even if ``explain`` is `True`.
1782 If a negative value is given a warning will be issued if the number
1783 of results is capped by that limit.
1784 explain : `bool`, optional
1785 If `True` (default) then `EmptyQueryResultError` exception is
1786 raised when resulting list is empty. The exception contains
1787 non-empty list of strings explaining possible causes for empty
1788 result.
1789 **kwargs
1790 Additional keyword arguments are forwarded to
1791 `DataCoordinate.standardize` when processing the ``data_id``
1792 argument (and may be used to provide a constraining data ID even
1793 when the ``data_id`` argument is `None`).
1795 Returns
1796 -------
1797 dataIds : `list` [`DataCoordinate`]
1798 Data IDs matching the given query parameters. These are always
1799 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1800 returns `True`).
1802 Raises
1803 ------
1804 lsst.daf.butler.registry.DataIdError
1805 Raised when ``data_id`` or keyword arguments specify unknown
1806 dimensions or values, or when they contain inconsistent values.
1807 lsst.daf.butler.registry.UserExpressionError
1808 Raised when ``where`` expression is invalid.
1809 lsst.daf.butler.EmptyQueryResultError
1810 Raised when query generates empty result and ``explain`` is set to
1811 `True`.
1812 TypeError
1813 Raised when the arguments are incompatible.
1814 """
1815 if data_id is None:
1816 data_id = DataCoordinate.make_empty(self.dimensions)
1817 if order_by is None:
1818 order_by = []
1819 query_limit = limit
1820 warn_limit = False
1821 if limit is not None and limit < 0:
1822 query_limit = abs(limit) + 1
1823 warn_limit = True
1824 with self.query() as query:
1825 result = (
1826 query.data_ids(dimensions)
1827 .where(data_id, where, bind=bind, **kwargs)
1828 .order_by(*ensure_iterable(order_by))
1829 .limit(query_limit)
1830 )
1831 if with_dimension_records:
1832 result = result.with_dimension_records()
1833 data_ids = list(result)
1834 if warn_limit and len(data_ids) == query_limit:
1835 # We asked for one too many so must remove that from the list.
1836 data_ids.pop(-1)
1837 assert limit is not None # For mypy.
1838 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1839 if explain and (limit is None or limit != 0) and not data_ids:
1840 raise EmptyQueryResultError(list(result.explain_no_results()))
1841 return data_ids
1843 def query_datasets(
1844 self,
1845 dataset_type: str | DatasetType,
1846 collections: str | Iterable[str] | None = None,
1847 *,
1848 find_first: bool = True,
1849 data_id: DataId | None = None,
1850 where: str = "",
1851 bind: Mapping[str, Any] | None = None,
1852 with_dimension_records: bool = False,
1853 order_by: Iterable[str] | str | None = None,
1854 limit: int | None = -20_000,
1855 explain: bool = True,
1856 **kwargs: Any,
1857 ) -> list[DatasetRef]:
1858 """Query for dataset references matching user-provided criteria.
1860 Parameters
1861 ----------
1862 dataset_type : `str` or `DatasetType`
1863 Dataset type object or name to search for.
1864 collections : collection expression, optional
1865 A collection name or iterable of collection names to search. If not
1866 provided, the default collections are used. Can be a wildcard if
1867 ``find_first`` is `False` (if find first is requested the order
1868 of collections matters and wildcards make the order indeterminate).
1869 See :ref:`daf_butler_collection_expressions` for more information.
1870 find_first : `bool`, optional
1871 If `True` (default), for each result data ID, only yield one
1872 `DatasetRef` of each `DatasetType`, from the first collection in
1873 which a dataset of that dataset type appears (according to the
1874 order of ``collections`` passed in). If `True`, ``collections``
1875 must not contain wildcards.
1876 data_id : `dict` or `DataCoordinate`, optional
1877 A data ID whose key-value pairs are used as equality constraints in
1878 the query.
1879 where : `str`, optional
1880 A string expression similar to a SQL WHERE clause. May involve any
1881 column of a dimension table or (as a shortcut for the primary key
1882 column of a dimension table) dimension name. See
1883 :ref:`daf_butler_dimension_expressions` for more information.
1884 bind : `~collections.abc.Mapping`, optional
1885 Mapping containing literal values that should be injected into the
1886 ``where`` expression, keyed by the identifiers they replace. Values
1887 of collection type can be expanded in some cases; see
1888 :ref:`daf_butler_dimension_expressions_identifiers` for more
1889 information.
1890 with_dimension_records : `bool`, optional
1891 If `True` (default is `False`) then returned data IDs will have
1892 dimension records.
1893 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1894 Names of the columns/dimensions to use for ordering returned data
1895 IDs. Column name can be prefixed with minus (``-``) to use
1896 descending ordering.
1897 limit : `int` or `None`, optional
1898 Upper limit on the number of returned records. `None` can be used
1899 if no limit is wanted. A limit of ``0`` means that the query will
1900 be executed and validated but no results will be returned. In this
1901 case there will be no exception even if ``explain`` is `True`.
1902 If a negative value is given a warning will be issued if the number
1903 of results is capped by that limit.
1904 explain : `bool`, optional
1905 If `True` (default) then `EmptyQueryResultError` exception is
1906 raised when resulting list is empty. The exception contains
1907 non-empty list of strings explaining possible causes for empty
1908 result.
1909 **kwargs
1910 Additional keyword arguments are forwarded to
1911 `DataCoordinate.standardize` when processing the ``data_id``
1912 argument (and may be used to provide a constraining data ID even
1913 when the ``data_id`` argument is `None`).
1915 Returns
1916 -------
1917 refs : `.queries.DatasetRefQueryResults`
1918 Dataset references matching the given query criteria. Nested data
1919 IDs are guaranteed to include values for all implied dimensions
1920 (i.e. `DataCoordinate.hasFull` will return `True`).
1922 Raises
1923 ------
1924 lsst.daf.butler.registry.DatasetTypeExpressionError
1925 Raised when ``dataset_type`` expression is invalid.
1926 lsst.daf.butler.registry.DataIdError
1927 Raised when ``data_id`` or keyword arguments specify unknown
1928 dimensions or values, or when they contain inconsistent values.
1929 lsst.daf.butler.registry.UserExpressionError
1930 Raised when ``where`` expression is invalid.
1931 lsst.daf.butler.EmptyQueryResultError
1932 Raised when query generates empty result and ``explain`` is set to
1933 `True`.
1934 TypeError
1935 Raised when the arguments are incompatible, such as when a
1936 collection wildcard is passed when ``find_first`` is `True`, or
1937 when ``collections`` is `None` and default butler collections are
1938 not defined.
1939 """
1940 if data_id is None:
1941 data_id = DataCoordinate.make_empty(self.dimensions)
1942 if order_by is None:
1943 order_by = []
1944 if collections and has_globs(collections):
1945 # Wild cards need to be expanded but can only be allowed if
1946 # find_first=False because expanding wildcards does not return
1947 # a guaranteed ordering. Querying collection registry to expand
1948 # collections when we do not have wildcards is expensive so only
1949 # do it if we need it.
1950 if find_first:
1951 raise InvalidQueryError(
1952 f"Can not use wildcards in collections when find_first=True (given {collections})"
1953 )
1954 collections = self.collections.query(collections)
1955 query_limit = limit
1956 warn_limit = False
1957 if limit is not None and limit < 0:
1958 query_limit = abs(limit) + 1
1959 warn_limit = True
1960 with self.query() as query:
1961 result = (
1962 query.datasets(dataset_type, collections=collections, find_first=find_first)
1963 .where(data_id, where, bind=bind, **kwargs)
1964 .order_by(*ensure_iterable(order_by))
1965 .limit(query_limit)
1966 )
1967 if with_dimension_records:
1968 result = result.with_dimension_records()
1969 refs = list(result)
1970 if warn_limit and len(refs) == query_limit:
1971 # We asked for one too many so must remove that from the list.
1972 refs.pop(-1)
1973 assert limit is not None # For mypy.
1974 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1975 if explain and (limit is None or limit != 0) and not refs:
1976 raise EmptyQueryResultError(list(result.explain_no_results()))
1977 return refs
1979 def query_dimension_records(
1980 self,
1981 element: str,
1982 *,
1983 data_id: DataId | None = None,
1984 where: str = "",
1985 bind: Mapping[str, Any] | None = None,
1986 order_by: Iterable[str] | str | None = None,
1987 limit: int | None = -20_000,
1988 explain: bool = True,
1989 **kwargs: Any,
1990 ) -> list[DimensionRecord]:
1991 """Query for dimension information matching user-provided criteria.
1993 Parameters
1994 ----------
1995 element : `str`
1996 The name of a dimension element to obtain records for.
1997 data_id : `dict` or `DataCoordinate`, optional
1998 A data ID whose key-value pairs are used as equality constraints
1999 in the query.
2000 where : `str`, optional
2001 A string expression similar to a SQL WHERE clause. See
2002 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
2003 information.
2004 bind : `~collections.abc.Mapping`, optional
2005 Mapping containing literal values that should be injected into the
2006 ``where`` expression, keyed by the identifiers they replace.
2007 Values of collection type can be expanded in some cases; see
2008 :ref:`daf_butler_dimension_expressions_identifiers` for more
2009 information.
2010 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2011 Names of the columns/dimensions to use for ordering returned data
2012 IDs. Column name can be prefixed with minus (``-``) to use
2013 descending ordering.
2014 limit : `int` or `None`, optional
2015 Upper limit on the number of returned records. `None` can be used
2016 if no limit is wanted. A limit of ``0`` means that the query will
2017 be executed and validated but no results will be returned. In this
2018 case there will be no exception even if ``explain`` is `True`.
2019 If a negative value is given a warning will be issued if the number
2020 of results is capped by that limit.
2021 explain : `bool`, optional
2022 If `True` (default) then `EmptyQueryResultError` exception is
2023 raised when resulting list is empty. The exception contains
2024 non-empty list of strings explaining possible causes for empty
2025 result.
2026 **kwargs
2027 Additional keyword arguments are forwarded to
2028 `DataCoordinate.standardize` when processing the ``data_id``
2029 argument (and may be used to provide a constraining data ID even
2030 when the ``data_id`` argument is `None`).
2032 Returns
2033 -------
2034 records : `list` [`DimensionRecord`]
2035 Dimension records matching the given query parameters.
2037 Raises
2038 ------
2039 lsst.daf.butler.registry.DataIdError
2040 Raised when ``data_id`` or keyword arguments specify unknown
2041 dimensions or values, or when they contain inconsistent values.
2042 lsst.daf.butler.registry.UserExpressionError
2043 Raised when ``where`` expression is invalid.
2044 lsst.daf.butler.EmptyQueryResultError
2045 Raised when query generates empty result and ``explain`` is set to
2046 `True`.
2047 TypeError
2048 Raised when the arguments are incompatible, such as when a
2049 collection wildcard is passed when ``find_first`` is `True`, or
2050 when ``collections`` is `None` and default butler collections are
2051 not defined.
2052 """
2053 if data_id is None:
2054 data_id = DataCoordinate.make_empty(self.dimensions)
2055 if order_by is None:
2056 order_by = []
2057 query_limit = limit
2058 warn_limit = False
2059 if limit is not None and limit < 0:
2060 query_limit = abs(limit) + 1
2061 warn_limit = True
2062 with self.query() as query:
2063 result = (
2064 query.dimension_records(element)
2065 .where(data_id, where, bind=bind, **kwargs)
2066 .order_by(*ensure_iterable(order_by))
2067 .limit(query_limit)
2068 )
2069 dimension_records = list(result)
2070 if warn_limit and len(dimension_records) == query_limit:
2071 # We asked for one too many so must remove that from the list.
2072 dimension_records.pop(-1)
2073 assert limit is not None # For mypy.
2074 _LOG.warning(
2075 "More dimension records are available than the requested limit of %d.", abs(limit)
2076 )
2077 if explain and (limit is None or limit != 0) and not dimension_records:
2078 raise EmptyQueryResultError(list(result.explain_no_results()))
2079 return dimension_records
2081 def query_all_datasets(
2082 self,
2083 collections: str | Iterable[str] | None = None,
2084 *,
2085 name: str | Iterable[str] = "*",
2086 find_first: bool = True,
2087 data_id: DataId | None = None,
2088 where: str = "",
2089 bind: Mapping[str, Any] | None = None,
2090 limit: int | None = -20_000,
2091 **kwargs: Any,
2092 ) -> list[DatasetRef]:
2093 """Query for datasets of potentially multiple types.
2095 Parameters
2096 ----------
2097 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2098 The collection or collections to search, in order. If not provided
2099 or `None`, the default collection search path for this butler is
2100 used.
2101 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2102 Names or name patterns (glob-style) that returned dataset type
2103 names must match. If an iterable, items are OR'd together. The
2104 default is to include all dataset types in the given collections.
2105 find_first : `bool`, optional
2106 If `True` (default), for each result data ID, only yield one
2107 `DatasetRef` of each `DatasetType`, from the first collection in
2108 which a dataset of that dataset type appears (according to the
2109 order of ``collections`` passed in).
2110 data_id : `dict` or `DataCoordinate`, optional
2111 A data ID whose key-value pairs are used as equality constraints in
2112 the query.
2113 where : `str`, optional
2114 A string expression similar to a SQL WHERE clause. May involve any
2115 column of a dimension table or (as a shortcut for the primary key
2116 column of a dimension table) dimension name. See
2117 :ref:`daf_butler_dimension_expressions` for more information.
2118 bind : `~collections.abc.Mapping`, optional
2119 Mapping containing literal values that should be injected into the
2120 ``where`` expression, keyed by the identifiers they replace. Values
2121 of collection type can be expanded in some cases; see
2122 :ref:`daf_butler_dimension_expressions_identifiers` for more
2123 information.
2124 limit : `int` or `None`, optional
2125 Upper limit on the number of returned records. `None` can be used
2126 if no limit is wanted. A limit of ``0`` means that the query will
2127 be executed and validated but no results will be returned.
2128 If a negative value is given a warning will be issued if the number
2129 of results is capped by that limit. If no limit is provided, by
2130 default a maximum of 20,000 records will be returned.
2131 **kwargs
2132 Additional keyword arguments are forwarded to
2133 `DataCoordinate.standardize` when processing the ``data_id``
2134 argument (and may be used to provide a constraining data ID even
2135 when the ``data_id`` argument is `None`).
2137 Raises
2138 ------
2139 MissingDatasetTypeError
2140 When no dataset types match ``name``, or an explicit (non-glob)
2141 dataset type in ``name`` does not exist.
2142 InvalidQueryError
2143 If the parameters to the query are inconsistent or malformed.
2144 MissingCollectionError
2145 If a given collection is not found.
2147 Returns
2148 -------
2149 refs : `list` [ `DatasetRef` ]
2150 Dataset references matching the given query criteria. Nested data
2151 IDs are guaranteed to include values for all implied dimensions
2152 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2153 include dimension records (`DataCoordinate.hasRecords` will be
2154 `False`).
2155 """
2156 if collections is None:
2157 collections = list(self.collections.defaults)
2158 else:
2159 collections = list(ensure_iterable(collections))
2161 if bind is None:
2162 bind = {}
2163 if data_id is None:
2164 data_id = {}
2166 warn_limit = False
2167 if limit is not None and limit < 0:
2168 # Add one to the limit so we can detect if we have exceeded it.
2169 limit = abs(limit) + 1
2170 warn_limit = True
2172 args = QueryAllDatasetsParameters(
2173 collections=collections,
2174 name=list(ensure_iterable(name)),
2175 find_first=find_first,
2176 data_id=data_id,
2177 where=where,
2178 limit=limit,
2179 bind=bind,
2180 kwargs=kwargs,
2181 with_dimension_records=False,
2182 )
2183 with self._query_all_datasets_by_page(args) as pages:
2184 result = []
2185 for page in pages:
2186 result.extend(page)
2188 if warn_limit and limit is not None and len(result) >= limit:
2189 # Remove the extra dataset we added for the limit check.
2190 result.pop()
2191 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2193 return result
2195 @abstractmethod
2196 def _query_all_datasets_by_page(
2197 self, args: QueryAllDatasetsParameters
2198 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2199 raise NotImplementedError()
2201 def clone(
2202 self,
2203 *,
2204 collections: CollectionArgType | None | EllipsisType = ...,
2205 run: str | None | EllipsisType = ...,
2206 inferDefaults: bool | EllipsisType = ...,
2207 dataId: dict[str, str] | EllipsisType = ...,
2208 metrics: ButlerMetrics | None = None,
2209 ) -> Butler:
2210 """Return a new Butler instance connected to the same repository
2211 as this one, optionally overriding ``collections``, ``run``,
2212 ``inferDefaults``, and default data ID.
2214 Parameters
2215 ----------
2216 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2217 optional
2218 Same as constructor. If omitted, uses value from original object.
2219 run : `str` or `None`, optional
2220 Same as constructor. If `None`, no default run is used. If
2221 omitted, copies value from original object.
2222 inferDefaults : `bool`, optional
2223 Same as constructor. If omitted, copies value from original
2224 object.
2225 dataId : `str`
2226 Same as ``kwargs`` passed to the constructor. If omitted, copies
2227 values from original object.
2228 metrics : `ButlerMetrics` or `None`, optional
2229 Metrics object to record butler statistics.
2230 """
2231 raise NotImplementedError()
2233 @abstractmethod
2234 def close(self) -> None:
2235 raise NotImplementedError()
2237 @abstractmethod
2238 def _expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
2239 raise NotImplementedError()