Coverage for python / lsst / daf / butler / _butler.py: 30%
310 statements
« prev ^ index » next coverage.py v7.13.2, created at 2026-02-04 02:35 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 metrics : `ButlerMetrics` or `None`
142 External metrics object to be used for tracking butler usage. If `None`
143 a new metrics object is created.
144 **kwargs : `typing.Any`
145 Additional keyword arguments passed to a constructor of actual butler
146 class.
148 Notes
149 -----
150 The preferred way to instantiate Butler is via the `from_config` method.
151 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
152 but ``mypy`` will complain about the former.
153 """
155 def __new__(
156 cls,
157 config: Config | ResourcePathExpression | None = None,
158 *,
159 collections: Any = None,
160 run: str | None = None,
161 searchPaths: Sequence[ResourcePathExpression] | None = None,
162 writeable: bool | None = None,
163 inferDefaults: bool = True,
164 without_datastore: bool = False,
165 metrics: ButlerMetrics | None = None,
166 **kwargs: Any,
167 ) -> Butler:
168 if cls is Butler:
169 return Butler.from_config(
170 config=config,
171 collections=collections,
172 run=run,
173 searchPaths=searchPaths,
174 writeable=writeable,
175 inferDefaults=inferDefaults,
176 without_datastore=without_datastore,
177 metrics=metrics,
178 **kwargs,
179 )
181 # Note: we do not pass any parameters to __new__, Python will pass them
182 # to __init__ after __new__ returns sub-class instance.
183 return super().__new__(cls)
185 @classmethod
186 def from_config(
187 cls,
188 config: Config | ResourcePathExpression | None = None,
189 *,
190 collections: Any = None,
191 run: str | None = None,
192 searchPaths: Sequence[ResourcePathExpression] | None = None,
193 writeable: bool | None = None,
194 inferDefaults: bool = True,
195 without_datastore: bool = False,
196 metrics: ButlerMetrics | None = None,
197 **kwargs: Any,
198 ) -> Butler:
199 """Create butler instance from configuration.
201 Parameters
202 ----------
203 config : `ButlerConfig`, `Config` or `str`, optional
204 Configuration. Anything acceptable to the `ButlerConfig`
205 constructor. If a directory path is given the configuration will be
206 read from a ``butler.yaml`` file in that location. If `None` is
207 given default values will be used. If ``config`` contains "cls" key
208 then its value is used as a name of butler class and it must be a
209 sub-class of this class, otherwise `DirectButler` is instantiated.
210 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
211 An expression specifying the collections to be searched (in order)
212 when reading datasets.
213 This may be a `str` collection name or an iterable thereof.
214 See :ref:`daf_butler_collection_expressions` for more information.
215 These collections are not registered automatically and must be
216 manually registered before they are used by any method, but they
217 may be manually registered after the `Butler` is initialized.
218 run : `str`, optional
219 Name of the `~CollectionType.RUN` collection new datasets should be
220 inserted into. If ``collections`` is `None` and ``run`` is not
221 `None`, ``collections`` will be set to ``[run]``. If not `None`,
222 this collection will automatically be registered. If this is not
223 set (and ``writeable`` is not set either), a read-only butler will
224 be created.
225 searchPaths : `list` of `str`, optional
226 Directory paths to search when calculating the full Butler
227 configuration. Not used if the supplied config is already a
228 `ButlerConfig`.
229 writeable : `bool`, optional
230 Explicitly sets whether the butler supports write operations. If
231 not provided, a read-write butler is created if any of ``run``,
232 ``tags``, or ``chains`` is non-empty.
233 inferDefaults : `bool`, optional
234 If `True` (default) infer default data ID values from the values
235 present in the datasets in ``collections``: if all collections have
236 the same value (or no value) for a governor dimension, that value
237 will be the default for that dimension. Nonexistent collections
238 are ignored. If a default value is provided explicitly for a
239 governor dimension via ``**kwargs``, no default will be inferred
240 for that dimension.
241 without_datastore : `bool`, optional
242 If `True` do not attach a datastore to this butler. Any attempts
243 to use a datastore will fail.
244 metrics : `ButlerMetrics` or `None`, optional
245 Metrics object to record butler usage statistics.
246 **kwargs : `typing.Any`
247 Default data ID key-value pairs. These may only identify
248 "governor" dimensions like ``instrument`` and ``skymap``.
250 Returns
251 -------
252 butler : `Butler`
253 A `Butler` constructed from the given configuration.
255 Notes
256 -----
257 Calling this factory method is identical to calling
258 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
259 complains about ``Butler()`` call.
261 Examples
262 --------
263 While there are many ways to control exactly how a `Butler` interacts
264 with the collections in its `Registry`, the most common cases are still
265 simple.
267 For a read-only `Butler` that searches one collection, do::
269 butler = Butler.from_config(
270 "/path/to/repo", collections=["u/alice/DM-50000"]
271 )
273 For a read-write `Butler` that writes to and reads from a
274 `~CollectionType.RUN` collection::
276 butler = Butler.from_config(
277 "/path/to/repo", run="u/alice/DM-50000/a"
278 )
280 The `Butler` passed to a ``PipelineTask`` is often much more complex,
281 because we want to write to one `~CollectionType.RUN` collection but
282 read from several others (as well)::
284 butler = Butler.from_config(
285 "/path/to/repo",
286 run="u/alice/DM-50000/a",
287 collections=[
288 "u/alice/DM-50000/a",
289 "u/bob/DM-49998",
290 "HSC/defaults",
291 ],
292 )
294 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
295 Datasets will be read first from that run (since it appears first in
296 the chain), and then from ``u/bob/DM-49998`` and finally
297 ``HSC/defaults``.
299 Finally, one can always create a `Butler` with no collections::
301 butler = Butler.from_config("/path/to/repo", writeable=True)
303 This can be extremely useful when you just want to use
304 ``butler.registry``, e.g. for inserting dimension data or managing
305 collections, or when the collections you want to use with the butler
306 are not consistent. Passing ``writeable`` explicitly here is only
307 necessary if you want to be able to make changes to the repo - usually
308 the value for ``writeable`` can be guessed from the collection
309 arguments provided, but it defaults to `False` when there are not
310 collection arguments.
311 """
312 # DirectButler used to have a way to specify a "copy constructor" by
313 # passing the "butler" parameter to its constructor. This has
314 # been moved out of the constructor into Butler.clone().
315 butler = kwargs.pop("butler", None)
316 metrics = metrics if metrics is not None else ButlerMetrics()
317 if butler is not None:
318 if not isinstance(butler, Butler):
319 raise TypeError("'butler' parameter must be a Butler instance")
320 if config is not None or searchPaths is not None or writeable is not None:
321 raise TypeError(
322 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
323 )
324 return butler.clone(
325 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
326 )
328 options = ButlerInstanceOptions(
329 collections=collections,
330 run=run,
331 writeable=writeable,
332 inferDefaults=inferDefaults,
333 metrics=metrics,
334 kwargs=kwargs,
335 )
337 # Load the Butler configuration. This may involve searching the
338 # environment to locate a configuration file.
339 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
340 butler_type = butler_config.get_butler_type()
342 # Make DirectButler if class is not specified.
343 match butler_type:
344 case ButlerType.DIRECT:
345 from .direct_butler import DirectButler
347 return DirectButler.create_from_config(
348 butler_config,
349 options=options,
350 without_datastore=without_datastore,
351 )
352 case ButlerType.REMOTE:
353 from .remote_butler._factory import RemoteButlerFactory
355 # Assume this is being created by a client who would like
356 # default caching of remote datasets.
357 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
358 return factory.create_butler_with_credentials_from_environment(
359 butler_options=options, enable_datastore_cache=True
360 )
361 case _:
362 raise TypeError(f"Unknown Butler type '{butler_type}'")
364 @staticmethod
365 def makeRepo(
366 root: ResourcePathExpression,
367 config: Config | str | None = None,
368 dimensionConfig: Config | str | None = None,
369 standalone: bool = False,
370 searchPaths: list[str] | None = None,
371 forceConfigRoot: bool = True,
372 outfile: ResourcePathExpression | None = None,
373 overwrite: bool = False,
374 ) -> Config:
375 """Create an empty data repository by adding a butler.yaml config
376 to a repository root directory.
378 Parameters
379 ----------
380 root : `lsst.resources.ResourcePathExpression`
381 Path or URI to the root location of the new repository. Will be
382 created if it does not exist.
383 config : `Config` or `str`, optional
384 Configuration to write to the repository, after setting any
385 root-dependent Registry or Datastore config options. Can not
386 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
387 configuration will be used. Root-dependent config options
388 specified in this config are overwritten if ``forceConfigRoot``
389 is `True`.
390 dimensionConfig : `Config` or `str`, optional
391 Configuration for dimensions, will be used to initialize registry
392 database.
393 standalone : `bool`
394 If True, write all expanded defaults, not just customized or
395 repository-specific settings.
396 This (mostly) decouples the repository from the default
397 configuration, insulating it from changes to the defaults (which
398 may be good or bad, depending on the nature of the changes).
399 Future *additions* to the defaults will still be picked up when
400 initializing `Butlers` to repos created with ``standalone=True``.
401 searchPaths : `list` of `str`, optional
402 Directory paths to search when calculating the full butler
403 configuration.
404 forceConfigRoot : `bool`, optional
405 If `False`, any values present in the supplied ``config`` that
406 would normally be reset are not overridden and will appear
407 directly in the output config. This allows non-standard overrides
408 of the root directory for a datastore or registry to be given.
409 If this parameter is `True` the values for ``root`` will be
410 forced into the resulting config if appropriate.
411 outfile : `lss.resources.ResourcePathExpression`, optional
412 If not-`None`, the output configuration will be written to this
413 location rather than into the repository itself. Can be a URI
414 string. Can refer to a directory that will be used to write
415 ``butler.yaml``.
416 overwrite : `bool`, optional
417 Create a new configuration file even if one already exists
418 in the specified output location. Default is to raise
419 an exception.
421 Returns
422 -------
423 config : `Config`
424 The updated `Config` instance written to the repo.
426 Raises
427 ------
428 ValueError
429 Raised if a ButlerConfig or ConfigSubset is passed instead of a
430 regular Config (as these subclasses would make it impossible to
431 support ``standalone=False``).
432 FileExistsError
433 Raised if the output config file already exists.
434 os.error
435 Raised if the directory does not exist, exists but is not a
436 directory, or cannot be created.
438 Notes
439 -----
440 Note that when ``standalone=False`` (the default), the configuration
441 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
442 construct the repository should also be used to construct any Butlers
443 to avoid configuration inconsistencies.
444 """
445 if isinstance(config, ButlerConfig | ConfigSubset):
446 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
448 # Ensure that the root of the repository exists or can be made
449 root_uri = ResourcePath(root, forceDirectory=True)
450 root_uri.mkdir()
452 config = Config(config)
454 # If we are creating a new repo from scratch with relative roots,
455 # do not propagate an explicit root from the config file
456 if "root" in config:
457 del config["root"]
459 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
460 imported_class = doImportType(full["datastore", "cls"])
461 if not issubclass(imported_class, Datastore):
462 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
463 datastoreClass: type[Datastore] = imported_class
464 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
466 # if key exists in given config, parse it, otherwise parse the defaults
467 # in the expanded config
468 if config.get(("registry", "db")):
469 registryConfig = RegistryConfig(config)
470 else:
471 registryConfig = RegistryConfig(full)
472 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
473 if defaultDatabaseUri is not None:
474 Config.updateParameters(
475 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
476 )
477 else:
478 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
480 if standalone:
481 config.merge(full)
482 else:
483 # Always expand the registry.managers section into the per-repo
484 # config, because after the database schema is created, it's not
485 # allowed to change anymore. Note that in the standalone=True
486 # branch, _everything_ in the config is expanded, so there's no
487 # need to special case this.
488 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
489 configURI: ResourcePathExpression
490 if outfile is not None:
491 # When writing to a separate location we must include
492 # the root of the butler repo in the config else it won't know
493 # where to look.
494 config["root"] = root_uri.geturl()
495 configURI = outfile
496 else:
497 configURI = root_uri
498 # Strip obscore configuration, if it is present, before writing config
499 # to a file, obscore config will be stored in registry.
500 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
501 config_to_write = config.copy()
502 del config_to_write[obscore_config_key]
503 config_to_write.dumpToUri(configURI, overwrite=overwrite)
504 # configFile attribute is updated, need to copy it to original.
505 config.configFile = config_to_write.configFile
506 else:
507 config.dumpToUri(configURI, overwrite=overwrite)
509 # Create Registry and populate tables
510 registryConfig = RegistryConfig(config.get("registry"))
511 dimensionConfig = DimensionConfig(dimensionConfig)
512 registry = _RegistryFactory(registryConfig).create_from_config(
513 dimensionConfig=dimensionConfig, butlerRoot=root_uri
514 )
515 registry.close()
517 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
519 return config
521 @classmethod
522 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
523 """Look up the label in a butler repository index.
525 Parameters
526 ----------
527 label : `str`
528 Label of the Butler repository to look up.
529 return_label : `bool`, optional
530 If ``label`` cannot be found in the repository index (either
531 because index is not defined or ``label`` is not in the index) and
532 ``return_label`` is `True` then return ``ResourcePath(label)``.
533 If ``return_label`` is `False` (default) then an exception will be
534 raised instead.
536 Returns
537 -------
538 uri : `lsst.resources.ResourcePath`
539 URI to the Butler repository associated with the given label or
540 default value if it is provided.
542 Raises
543 ------
544 KeyError
545 Raised if the label is not found in the index, or if an index
546 is not defined, and ``return_label`` is `False`.
548 Notes
549 -----
550 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
551 information is discovered.
552 """
553 return ButlerRepoIndex.get_repo_uri(label, return_label)
555 @classmethod
556 def get_known_repos(cls) -> set[str]:
557 """Retrieve the list of known repository labels.
559 Returns
560 -------
561 repos : `set` of `str`
562 All the known labels. Can be empty if no index can be found.
564 Notes
565 -----
566 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
567 information is discovered.
568 """
569 return ButlerRepoIndex.get_known_repos()
571 @classmethod
572 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
573 """Extract the butler label and dataset ID from a dataset URI.
575 Parameters
576 ----------
577 uri : `str`
578 The dataset URI to parse.
580 Returns
581 -------
582 parsed : `ParsedButlerDatasetURI`
583 The label associated with the butler repository from which this
584 dataset originates and the ID of the dataset.
586 Notes
587 -----
588 Supports dataset URIs of the forms
589 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
590 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
591 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
592 can include anything supported by the `Butler` constructor, including
593 paths to repositories and alias labels.
595 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
597 will return a label of ``/repo/main``.
599 This method does not attempt to check that the dataset exists in the
600 labeled butler.
602 Since the IVOID can be issued by any publisher to represent a Butler
603 dataset there is no validation of the path or netloc component of the
604 URI. The only requirement is that there are ``id`` and ``repo`` keys
605 in the ``ivo`` URI query component.
606 """
607 parsed = urllib.parse.urlparse(uri)
608 parsed_scheme = parsed.scheme.lower()
609 if parsed_scheme == "ivo":
610 # Do not validate the netloc or the path values.
611 qs = urllib.parse.parse_qs(parsed.query)
612 if "repo" not in qs or "id" not in qs:
613 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
614 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
615 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
616 label = qs["repo"][0]
617 id_ = qs["id"][0]
618 elif parsed_scheme == "butler":
619 label = parsed.netloc # Butler label is case sensitive.
620 # Need to strip the leading /.
621 id_ = parsed.path[1:]
622 else:
623 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
624 # Strip trailing/leading whitespace from label.
625 label = label.strip()
626 if not label:
627 raise ValueError(f"No butler repository label found in uri {uri!r}")
628 try:
629 dataset_id = uuid.UUID(hex=id_)
630 except Exception as e:
631 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
632 raise
634 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
636 @classmethod
637 def get_dataset_from_uri(
638 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
639 ) -> SpecificButlerDataset:
640 """Get the dataset associated with the given dataset URI.
642 Parameters
643 ----------
644 uri : `str`
645 The URI associated with a dataset.
646 factory : `LabeledButlerFactoryProtocol` or `None`, optional
647 Bound factory function that will be given the butler label
648 and receive a `Butler`. If this is not provided the label
649 will be tried directly.
651 Returns
652 -------
653 result : `SpecificButlerDataset`
654 The butler associated with this URI and the dataset itself.
655 The dataset can be `None` if the UUID is valid but the dataset
656 is not known to this butler.
657 """
658 parsed = cls.parse_dataset_uri(uri)
659 butler: Butler | None = None
660 if factory is not None:
661 # If the label is not recognized, it might be a path.
662 try:
663 butler = factory(parsed.label)
664 except KeyError:
665 pass
666 if butler is None:
667 butler = cls.from_config(parsed.label)
668 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
670 @abstractmethod
671 def _caching_context(self) -> AbstractContextManager[None]:
672 """Context manager that enables caching."""
673 raise NotImplementedError()
675 @abstractmethod
676 def transaction(self) -> AbstractContextManager[None]:
677 """Context manager supporting `Butler` transactions.
679 Transactions can be nested.
680 """
681 raise NotImplementedError()
683 @abstractmethod
684 def put(
685 self,
686 obj: Any,
687 datasetRefOrType: DatasetRef | DatasetType | str,
688 /,
689 dataId: DataId | None = None,
690 *,
691 run: str | None = None,
692 provenance: DatasetProvenance | None = None,
693 **kwargs: Any,
694 ) -> DatasetRef:
695 """Store and register a dataset.
697 Parameters
698 ----------
699 obj : `object`
700 The dataset.
701 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
702 When `DatasetRef` is provided, ``dataId`` should be `None`.
703 Otherwise the `DatasetType` or name thereof. If a fully resolved
704 `DatasetRef` is given the run and ID are used directly.
705 dataId : `dict` or `DataCoordinate`
706 A `dict` of `Dimension` link name, value pairs that label the
707 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
708 should be provided as the second argument.
709 run : `str`, optional
710 The name of the run the dataset should be added to, overriding
711 ``self.run``. Not used if a resolved `DatasetRef` is provided.
712 provenance : `DatasetProvenance` or `None`, optional
713 Any provenance that should be attached to the serialized dataset.
714 Not supported by all serialization mechanisms.
715 **kwargs
716 Additional keyword arguments used to augment or construct a
717 `DataCoordinate`. See `DataCoordinate.standardize`
718 parameters. Not used if a resolve `DatasetRef` is provided.
720 Returns
721 -------
722 ref : `DatasetRef`
723 A reference to the stored dataset, updated with the correct id if
724 given.
726 Raises
727 ------
728 TypeError
729 Raised if the butler is read-only or if no run has been provided.
730 """
731 raise NotImplementedError()
733 @abstractmethod
734 def getDeferred(
735 self,
736 datasetRefOrType: DatasetRef | DatasetType | str,
737 /,
738 dataId: DataId | None = None,
739 *,
740 parameters: dict | None = None,
741 collections: Any = None,
742 storageClass: str | StorageClass | None = None,
743 timespan: Timespan | None = None,
744 **kwargs: Any,
745 ) -> DeferredDatasetHandle:
746 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
747 after an immediate registry lookup.
749 Parameters
750 ----------
751 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
752 When `DatasetRef` the `dataId` should be `None`.
753 Otherwise the `DatasetType` or name thereof.
754 dataId : `dict` or `DataCoordinate`, optional
755 A `dict` of `Dimension` link name, value pairs that label the
756 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
757 should be provided as the first argument.
758 parameters : `dict`
759 Additional StorageClass-defined options to control reading,
760 typically used to efficiently read only a subset of the dataset.
761 collections : Any, optional
762 Collections to be searched, overriding ``self.collections``.
763 Can be any of the types supported by the ``collections`` argument
764 to butler construction.
765 storageClass : `StorageClass` or `str`, optional
766 The storage class to be used to override the Python type
767 returned by this method. By default the returned type matches
768 the dataset type definition for this dataset. Specifying a
769 read `StorageClass` can force a different type to be returned.
770 This type must be compatible with the original type.
771 timespan : `Timespan` or `None`, optional
772 A timespan that the validity range of the dataset must overlap.
773 If not provided and this is a calibration dataset type, an attempt
774 will be made to find the timespan from any temporal coordinate
775 in the data ID.
776 **kwargs
777 Additional keyword arguments used to augment or construct a
778 `DataId`. See `DataId` parameters.
780 Returns
781 -------
782 obj : `DeferredDatasetHandle`
783 A handle which can be used to retrieve a dataset at a later time.
785 Raises
786 ------
787 LookupError
788 Raised if no matching dataset exists in the `Registry` or
789 datastore.
790 ValueError
791 Raised if a resolved `DatasetRef` was passed as an input, but it
792 differs from the one found in the registry.
793 TypeError
794 Raised if no collections were provided.
795 """
796 raise NotImplementedError()
798 @abstractmethod
799 def get(
800 self,
801 datasetRefOrType: DatasetRef | DatasetType | str,
802 /,
803 dataId: DataId | None = None,
804 *,
805 parameters: dict[str, Any] | None = None,
806 collections: Any = None,
807 storageClass: StorageClass | str | None = None,
808 timespan: Timespan | None = None,
809 **kwargs: Any,
810 ) -> Any:
811 """Retrieve a stored dataset.
813 Parameters
814 ----------
815 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
816 When `DatasetRef` the `dataId` should be `None`.
817 Otherwise the `DatasetType` or name thereof.
818 If a resolved `DatasetRef`, the associated dataset
819 is returned directly without additional querying.
820 dataId : `dict` or `DataCoordinate`
821 A `dict` of `Dimension` link name, value pairs that label the
822 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
823 should be provided as the first argument.
824 parameters : `dict`
825 Additional StorageClass-defined options to control reading,
826 typically used to efficiently read only a subset of the dataset.
827 collections : Any, optional
828 Collections to be searched, overriding ``self.collections``.
829 Can be any of the types supported by the ``collections`` argument
830 to butler construction.
831 storageClass : `StorageClass` or `str`, optional
832 The storage class to be used to override the Python type
833 returned by this method. By default the returned type matches
834 the dataset type definition for this dataset. Specifying a
835 read `StorageClass` can force a different type to be returned.
836 This type must be compatible with the original type.
837 timespan : `Timespan` or `None`, optional
838 A timespan that the validity range of the dataset must overlap.
839 If not provided and this is a calibration dataset type, an attempt
840 will be made to find the timespan from any temporal coordinate
841 in the data ID.
842 **kwargs
843 Additional keyword arguments used to augment or construct a
844 `DataCoordinate`. See `DataCoordinate.standardize`
845 parameters.
847 Returns
848 -------
849 obj : `object`
850 The dataset.
852 Raises
853 ------
854 LookupError
855 Raised if no matching dataset exists in the `Registry`.
856 TypeError
857 Raised if no collections were provided.
859 Notes
860 -----
861 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
862 this method requires that the given data ID include temporal dimensions
863 beyond the dimensions of the dataset type itself, in order to find the
864 dataset with the appropriate validity range. For example, a "bias"
865 dataset with native dimensions ``{instrument, detector}`` could be
866 fetched with a ``{instrument, detector, exposure}`` data ID, because
867 ``exposure`` is a temporal dimension.
868 """
869 raise NotImplementedError()
871 @abstractmethod
872 def getURIs(
873 self,
874 datasetRefOrType: DatasetRef | DatasetType | str,
875 /,
876 dataId: DataId | None = None,
877 *,
878 predict: bool = False,
879 collections: Any = None,
880 run: str | None = None,
881 **kwargs: Any,
882 ) -> DatasetRefURIs:
883 """Return the URIs associated with the dataset.
885 Parameters
886 ----------
887 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
888 When `DatasetRef` the `dataId` should be `None`.
889 Otherwise the `DatasetType` or name thereof.
890 dataId : `dict` or `DataCoordinate`
891 A `dict` of `Dimension` link name, value pairs that label the
892 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
893 should be provided as the first argument.
894 predict : `bool`
895 If `True`, allow URIs to be returned of datasets that have not
896 been written.
897 collections : Any, optional
898 Collections to be searched, overriding ``self.collections``.
899 Can be any of the types supported by the ``collections`` argument
900 to butler construction.
901 run : `str`, optional
902 Run to use for predictions, overriding ``self.run``.
903 **kwargs
904 Additional keyword arguments used to augment or construct a
905 `DataCoordinate`. See `DataCoordinate.standardize`
906 parameters.
908 Returns
909 -------
910 uris : `DatasetRefURIs`
911 The URI to the primary artifact associated with this dataset (if
912 the dataset was disassembled within the datastore this may be
913 `None`), and the URIs to any components associated with the dataset
914 artifact. (can be empty if there are no components).
915 """
916 raise NotImplementedError()
918 def getURI(
919 self,
920 datasetRefOrType: DatasetRef | DatasetType | str,
921 /,
922 dataId: DataId | None = None,
923 *,
924 predict: bool = False,
925 collections: Any = None,
926 run: str | None = None,
927 **kwargs: Any,
928 ) -> ResourcePath:
929 """Return the URI to the Dataset.
931 Parameters
932 ----------
933 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
934 When `DatasetRef` the `dataId` should be `None`.
935 Otherwise the `DatasetType` or name thereof.
936 dataId : `dict` or `DataCoordinate`
937 A `dict` of `Dimension` link name, value pairs that label the
938 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
939 should be provided as the first argument.
940 predict : `bool`
941 If `True`, allow URIs to be returned of datasets that have not
942 been written.
943 collections : Any, optional
944 Collections to be searched, overriding ``self.collections``.
945 Can be any of the types supported by the ``collections`` argument
946 to butler construction.
947 run : `str`, optional
948 Run to use for predictions, overriding ``self.run``.
949 **kwargs
950 Additional keyword arguments used to augment or construct a
951 `DataCoordinate`. See `DataCoordinate.standardize`
952 parameters.
954 Returns
955 -------
956 uri : `lsst.resources.ResourcePath`
957 URI pointing to the Dataset within the datastore. If the
958 Dataset does not exist in the datastore, and if ``predict`` is
959 `True`, the URI will be a prediction and will include a URI
960 fragment "#predicted".
961 If the datastore does not have entities that relate well
962 to the concept of a URI the returned URI string will be
963 descriptive. The returned URI is not guaranteed to be obtainable.
965 Raises
966 ------
967 LookupError
968 A URI has been requested for a dataset that does not exist and
969 guessing is not allowed.
970 ValueError
971 Raised if a resolved `DatasetRef` was passed as an input, but it
972 differs from the one found in the registry.
973 TypeError
974 Raised if no collections were provided.
975 RuntimeError
976 Raised if a URI is requested for a dataset that consists of
977 multiple artifacts.
978 """
979 primary, components = self.getURIs(
980 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
981 )
983 if primary is None or components:
984 raise RuntimeError(
985 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
986 "Use Butler.getURIs() instead."
987 )
988 return primary
990 @abstractmethod
991 def get_dataset_type(self, name: str) -> DatasetType:
992 """Get the `DatasetType`.
994 Parameters
995 ----------
996 name : `str`
997 Name of the type.
999 Returns
1000 -------
1001 type : `DatasetType`
1002 The `DatasetType` associated with the given name.
1004 Raises
1005 ------
1006 lsst.daf.butler.MissingDatasetTypeError
1007 Raised if the requested dataset type has not been registered.
1009 Notes
1010 -----
1011 This method handles component dataset types automatically, though most
1012 other operations do not.
1013 """
1014 raise NotImplementedError()
1016 @abstractmethod
1017 def get_dataset(
1018 self,
1019 id: DatasetId | str,
1020 *,
1021 storage_class: str | StorageClass | None = None,
1022 dimension_records: bool = False,
1023 datastore_records: bool = False,
1024 ) -> DatasetRef | None:
1025 """Retrieve a Dataset entry.
1027 Parameters
1028 ----------
1029 id : `DatasetId`
1030 The unique identifier for the dataset, as an instance of
1031 `uuid.UUID` or a string containing a hexadecimal number.
1032 storage_class : `str` or `StorageClass` or `None`
1033 A storage class to use when creating the returned entry. If given
1034 it must be compatible with the default storage class.
1035 dimension_records : `bool`, optional
1036 If `True` the ref will be expanded and contain dimension records.
1037 datastore_records : `bool`, optional
1038 If `True` the ref will contain associated datastore records.
1040 Returns
1041 -------
1042 ref : `DatasetRef` or `None`
1043 A ref to the Dataset, or `None` if no matching Dataset
1044 was found.
1045 """
1046 raise NotImplementedError()
1048 @abstractmethod
1049 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1050 """Retrieve a list of dataset entries.
1052 Parameters
1053 ----------
1054 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1055 The unique identifiers for the datasets, as instances of
1056 `uuid.UUID` or strings containing a hexadecimal number.
1058 Returns
1059 -------
1060 refs : `list` [ `DatasetRef` ]
1061 A list containing a `DatasetRef` for each of the given dataset IDs.
1062 If a dataset was not found, no error is thrown -- it is just not
1063 included in the list. The returned datasets are in no particular
1064 order.
1065 """
1066 raise NotImplementedError()
1068 @abstractmethod
1069 def find_dataset(
1070 self,
1071 dataset_type: DatasetType | str,
1072 data_id: DataId | None = None,
1073 *,
1074 collections: str | Sequence[str] | None = None,
1075 timespan: Timespan | None = None,
1076 storage_class: str | StorageClass | None = None,
1077 dimension_records: bool = False,
1078 datastore_records: bool = False,
1079 **kwargs: Any,
1080 ) -> DatasetRef | None:
1081 """Find a dataset given its `DatasetType` and data ID.
1083 This can be used to obtain a `DatasetRef` that permits the dataset to
1084 be read from a `Datastore`. If the dataset is a component and can not
1085 be found using the provided dataset type, a dataset ref for the parent
1086 will be returned instead but with the correct dataset type.
1088 Parameters
1089 ----------
1090 dataset_type : `DatasetType` or `str`
1091 A `DatasetType` or the name of one. If this is a `DatasetType`
1092 instance, its storage class will be respected and propagated to
1093 the output, even if it differs from the dataset type definition
1094 in the registry, as long as the storage classes are convertible.
1095 data_id : `dict` or `DataCoordinate`, optional
1096 A `dict`-like object containing the `Dimension` links that identify
1097 the dataset within a collection. If it is a `dict` the dataId
1098 can include dimension record values such as ``day_obs`` and
1099 ``seq_num`` or ``full_name`` that can be used to derive the
1100 primary dimension.
1101 collections : `str` or `list` [`str`], optional
1102 A an ordered list of collections to search for the dataset.
1103 Defaults to ``self.defaults.collections``.
1104 timespan : `Timespan`, optional
1105 A timespan that the validity range of the dataset must overlap.
1106 If not provided, any `~CollectionType.CALIBRATION` collections
1107 matched by the ``collections`` argument will not be searched.
1108 storage_class : `str` or `StorageClass` or `None`
1109 A storage class to use when creating the returned entry. If given
1110 it must be compatible with the default storage class.
1111 dimension_records : `bool`, optional
1112 If `True` the ref will be expanded and contain dimension records.
1113 datastore_records : `bool`, optional
1114 If `True` the ref will contain associated datastore records.
1115 **kwargs
1116 Additional keyword arguments passed to
1117 `DataCoordinate.standardize` to convert ``dataId`` to a true
1118 `DataCoordinate` or augment an existing one. This can also include
1119 dimension record metadata that can be used to derive a primary
1120 dimension value.
1122 Returns
1123 -------
1124 ref : `DatasetRef`
1125 A reference to the dataset, or `None` if no matching Dataset
1126 was found.
1128 Raises
1129 ------
1130 lsst.daf.butler.NoDefaultCollectionError
1131 Raised if ``collections`` is `None` and
1132 ``self.collections`` is `None`.
1133 LookupError
1134 Raised if one or more data ID keys are missing.
1135 lsst.daf.butler.MissingDatasetTypeError
1136 Raised if the dataset type does not exist.
1137 lsst.daf.butler.MissingCollectionError
1138 Raised if any of ``collections`` does not exist in the registry.
1140 Notes
1141 -----
1142 This method simply returns `None` and does not raise an exception even
1143 when the set of collections searched is intrinsically incompatible with
1144 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1145 only `~CollectionType.CALIBRATION` collections are being searched.
1146 This may make it harder to debug some lookup failures, but the behavior
1147 is intentional; we consider it more important that failed searches are
1148 reported consistently, regardless of the reason, and that adding
1149 additional collections that do not contain a match to the search path
1150 never changes the behavior.
1152 This method handles component dataset types automatically, though most
1153 other query operations do not.
1154 """
1155 raise NotImplementedError()
1157 @abstractmethod
1158 def retrieve_artifacts_zip(
1159 self,
1160 refs: Iterable[DatasetRef],
1161 destination: ResourcePathExpression,
1162 overwrite: bool = True,
1163 ) -> ResourcePath:
1164 """Retrieve artifacts from a Butler and place in ZIP file.
1166 Parameters
1167 ----------
1168 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1169 The datasets to be included in the zip file.
1170 destination : `lsst.resources.ResourcePathExpression`
1171 Directory to write the new ZIP file. This directory will
1172 also be used as a staging area for the datasets being downloaded
1173 from the datastore.
1174 overwrite : `bool`, optional
1175 If `False` the output Zip will not be written if a file of the
1176 same name is already present in ``destination``.
1178 Returns
1179 -------
1180 zip_file : `lsst.resources.ResourcePath`
1181 The path to the new ZIP file.
1183 Raises
1184 ------
1185 ValueError
1186 Raised if there are no refs to retrieve.
1187 """
1188 raise NotImplementedError()
1190 @abstractmethod
1191 def retrieveArtifacts(
1192 self,
1193 refs: Iterable[DatasetRef],
1194 destination: ResourcePathExpression,
1195 transfer: str = "auto",
1196 preserve_path: bool = True,
1197 overwrite: bool = False,
1198 ) -> list[ResourcePath]:
1199 """Retrieve the artifacts associated with the supplied refs.
1201 Parameters
1202 ----------
1203 refs : iterable of `DatasetRef`
1204 The datasets for which artifacts are to be retrieved.
1205 A single ref can result in multiple artifacts. The refs must
1206 be resolved.
1207 destination : `lsst.resources.ResourcePath` or `str`
1208 Location to write the artifacts.
1209 transfer : `str`, optional
1210 Method to use to transfer the artifacts. Must be one of the options
1211 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1212 "move" is not allowed.
1213 preserve_path : `bool`, optional
1214 If `True` the full path of the artifact within the datastore
1215 is preserved. If `False` the final file component of the path
1216 is used.
1217 overwrite : `bool`, optional
1218 If `True` allow transfers to overwrite existing files at the
1219 destination.
1221 Returns
1222 -------
1223 targets : `list` of `lsst.resources.ResourcePath`
1224 URIs of file artifacts in destination location. Order is not
1225 preserved.
1227 Notes
1228 -----
1229 For non-file datastores the artifacts written to the destination
1230 may not match the representation inside the datastore. For example
1231 a hierarchical data structure in a NoSQL database may well be stored
1232 as a JSON file.
1233 """
1234 raise NotImplementedError()
1236 @abstractmethod
1237 def exists(
1238 self,
1239 dataset_ref_or_type: DatasetRef | DatasetType | str,
1240 /,
1241 data_id: DataId | None = None,
1242 *,
1243 full_check: bool = True,
1244 collections: Any = None,
1245 **kwargs: Any,
1246 ) -> DatasetExistence:
1247 """Indicate whether a dataset is known to Butler registry and
1248 datastore.
1250 Parameters
1251 ----------
1252 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1253 When `DatasetRef` the `dataId` should be `None`.
1254 Otherwise the `DatasetType` or name thereof.
1255 data_id : `dict` or `DataCoordinate`
1256 A `dict` of `Dimension` link name, value pairs that label the
1257 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1258 should be provided as the first argument.
1259 full_check : `bool`, optional
1260 If `True`, a check will be made for the actual existence of a
1261 dataset artifact. This will involve additional overhead due to
1262 the need to query an external system. If `False`, this check will
1263 be omitted, and the registry and datastore will solely be asked
1264 if they know about the dataset but no direct check for the
1265 artifact will be performed.
1266 collections : Any, optional
1267 Collections to be searched, overriding ``self.collections``.
1268 Can be any of the types supported by the ``collections`` argument
1269 to butler construction.
1270 **kwargs
1271 Additional keyword arguments used to augment or construct a
1272 `DataCoordinate`. See `DataCoordinate.standardize`
1273 parameters.
1275 Returns
1276 -------
1277 existence : `DatasetExistence`
1278 Object indicating whether the dataset is known to registry and
1279 datastore. Evaluates to `True` if the dataset is present and known
1280 to both.
1281 """
1282 raise NotImplementedError()
1284 @abstractmethod
1285 def _exists_many(
1286 self,
1287 refs: Iterable[DatasetRef],
1288 /,
1289 *,
1290 full_check: bool = True,
1291 ) -> dict[DatasetRef, DatasetExistence]:
1292 """Indicate whether multiple datasets are known to Butler registry and
1293 datastore.
1295 This is an experimental API that may change at any moment.
1297 Parameters
1298 ----------
1299 refs : iterable of `DatasetRef`
1300 The datasets to be checked.
1301 full_check : `bool`, optional
1302 If `True`, a check will be made for the actual existence of each
1303 dataset artifact. This will involve additional overhead due to
1304 the need to query an external system. If `False`, this check will
1305 be omitted, and the registry and datastore will solely be asked
1306 if they know about the dataset(s) but no direct check for the
1307 artifact(s) will be performed.
1309 Returns
1310 -------
1311 existence : dict of [`DatasetRef`, `DatasetExistence`]
1312 Mapping from the given dataset refs to an enum indicating the
1313 status of the dataset in registry and datastore.
1314 Each value evaluates to `True` if the dataset is present and known
1315 to both.
1316 """
1317 raise NotImplementedError()
1319 @abstractmethod
1320 def removeRuns(
1321 self,
1322 names: Iterable[str],
1323 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1324 *,
1325 unlink_from_chains: bool = False,
1326 ) -> None:
1327 """Remove one or more `~CollectionType.RUN` collections and the
1328 datasets within them.
1330 Parameters
1331 ----------
1332 names : `~collections.abc.Iterable` [ `str` ]
1333 The names of the collections to remove.
1334 unstore : `bool`, optional
1335 If `True` (default), delete datasets from all datastores in which
1336 they are present, and attempt to rollback the registry deletions if
1337 datastore deletions fail (which may not always be possible). If
1338 `False`, datastore records for these datasets are still removed,
1339 but any artifacts (e.g. files) will not be. This parameter is now
1340 deprecated and no longer has any effect. Files are always deleted
1341 from datastores unless they were ingested using full URIs.
1342 unlink_from_chains : `bool`, optional
1343 If `True` remove the RUN collection from any chains prior to
1344 removing the RUN. If `False` the removal will fail if any chains
1345 still refer to the RUN.
1347 Raises
1348 ------
1349 TypeError
1350 Raised if one or more collections are not of type
1351 `~CollectionType.RUN`.
1352 """
1353 raise NotImplementedError()
1355 @abstractmethod
1356 def ingest(
1357 self,
1358 *datasets: FileDataset,
1359 transfer: str | None = "auto",
1360 record_validation_info: bool = True,
1361 skip_existing: bool = False,
1362 ) -> None:
1363 """Store and register one or more datasets that already exist on disk.
1365 Parameters
1366 ----------
1367 *datasets : `FileDataset`
1368 Each positional argument is a struct containing information about
1369 a file to be ingested, including its URI (either absolute or
1370 relative to the datastore root, if applicable), a resolved
1371 `DatasetRef`, and optionally a formatter class or its
1372 fully-qualified string name. If a formatter is not provided, the
1373 formatter that would be used for `put` is assumed. On successful
1374 ingest all `FileDataset.formatter` attributes will be set to the
1375 formatter class used. `FileDataset.path` attributes may be modified
1376 to put paths in whatever the datastore considers a standardized
1377 form.
1378 transfer : `str`, optional
1379 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1380 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1381 transfer the file.
1382 record_validation_info : `bool`, optional
1383 If `True`, the default, the datastore can record validation
1384 information associated with the file. If `False` the datastore
1385 will not attempt to track any information such as checksums
1386 or file sizes. This can be useful if such information is tracked
1387 in an external system or if the file is to be compressed in place.
1388 It is up to the datastore whether this parameter is relevant.
1389 skip_existing : `bool`, optional
1390 If `True`, a dataset will not be ingested if a dataset with the
1391 same dataset ID already exists in the datastore.
1392 If `False` (the default), a `ConflictingDefinitionError` will be
1393 raised if any datasets with the same dataset ID already exist
1394 in the datastore.
1396 Returns
1397 -------
1398 None
1400 Raises
1401 ------
1402 TypeError
1403 Raised if the butler is read-only or if no run was provided.
1404 NotImplementedError
1405 Raised if the `Datastore` does not support the given transfer mode.
1406 DatasetTypeNotSupportedError
1407 Raised if one or more files to be ingested have a dataset type that
1408 is not supported by the `Datastore`..
1409 FileNotFoundError
1410 Raised if one of the given files does not exist.
1411 FileExistsError
1412 Raised if transfer is not `None` but the (internal) location the
1413 file would be moved to is already occupied.
1414 ConflictingDefinitionError
1415 Raised if a dataset already exists in the repository and
1416 ``skip_existing`` is `False`.
1418 Notes
1419 -----
1420 This operation is not fully exception safe: if a database operation
1421 fails, the given `FileDataset` instances may be only partially updated.
1423 It is atomic in terms of database operations (they will either all
1424 succeed or all fail) providing the database engine implements
1425 transactions correctly. It will attempt to be atomic in terms of
1426 filesystem operations as well, but this cannot be implemented
1427 rigorously for most datastores.
1428 """
1429 raise NotImplementedError()
1431 @abstractmethod
1432 def ingest_zip(
1433 self,
1434 zip_file: ResourcePathExpression,
1435 transfer: str = "auto",
1436 *,
1437 transfer_dimensions: bool = False,
1438 dry_run: bool = False,
1439 skip_existing: bool = False,
1440 ) -> None:
1441 """Ingest a Zip file into this butler.
1443 The Zip file must have been created by `retrieve_artifacts_zip`.
1445 Parameters
1446 ----------
1447 zip_file : `lsst.resources.ResourcePathExpression`
1448 Path to the Zip file.
1449 transfer : `str`, optional
1450 Method to use to transfer the Zip into the datastore.
1451 transfer_dimensions : `bool`, optional
1452 If `True`, dimension record data associated with the new datasets
1453 will be transferred from the Zip file, if present.
1454 dry_run : `bool`, optional
1455 If `True` the ingest will be processed without any modifications
1456 made to the target butler and as if the target butler did not
1457 have any of the datasets.
1458 skip_existing : `bool`, optional
1459 If `True`, a zip will not be ingested if the dataset entries listed
1460 in the index with the same dataset ID already exists in the butler.
1461 If `False` (the default), a `ConflictingDefinitionError` will be
1462 raised if any datasets with the same dataset ID already exist
1463 in the repository. If, somehow, some datasets are known to the
1464 butler and some are not, this is currently treated as an error
1465 rather than attempting to do a partial ingest.
1467 Notes
1468 -----
1469 Run collections and dataset types are created as needed.
1470 """
1471 raise NotImplementedError()
1473 @abstractmethod
1474 def export(
1475 self,
1476 *,
1477 directory: str | None = None,
1478 filename: str | None = None,
1479 format: str | None = None,
1480 transfer: str | None = None,
1481 ) -> AbstractContextManager[RepoExportContext]:
1482 """Export datasets from the repository represented by this `Butler`.
1484 This method is a context manager that returns a helper object
1485 (`RepoExportContext`) that is used to indicate what information from
1486 the repository should be exported.
1488 Parameters
1489 ----------
1490 directory : `str`, optional
1491 Directory dataset files should be written to if ``transfer`` is not
1492 `None`.
1493 filename : `str`, optional
1494 Name for the file that will include database information associated
1495 with the exported datasets. If this is not an absolute path and
1496 ``directory`` is not `None`, it will be written to ``directory``
1497 instead of the current working directory. Defaults to
1498 "export.{format}".
1499 format : `str`, optional
1500 File format for the database information file. If `None`, the
1501 extension of ``filename`` will be used.
1502 transfer : `str`, optional
1503 Transfer mode passed to `Datastore.export`.
1505 Raises
1506 ------
1507 TypeError
1508 Raised if the set of arguments passed is inconsistent.
1510 Examples
1511 --------
1512 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1513 methods are used to provide the iterables over data IDs and/or datasets
1514 to be exported::
1516 with butler.export("exports.yaml") as export:
1517 # Export all flats, but none of the dimension element rows
1518 # (i.e. data ID information) associated with them.
1519 export.saveDatasets(
1520 butler.registry.queryDatasets("flat"), elements=()
1521 )
1522 # Export all datasets that start with "deepCoadd_" and all of
1523 # their associated data ID information.
1524 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1525 """
1526 raise NotImplementedError()
1528 @abstractmethod
1529 def import_(
1530 self,
1531 *,
1532 directory: ResourcePathExpression | None = None,
1533 filename: ResourcePathExpression | TextIO | None = None,
1534 format: str | None = None,
1535 transfer: str | None = None,
1536 skip_dimensions: set | None = None,
1537 record_validation_info: bool = True,
1538 without_datastore: bool = False,
1539 ) -> None:
1540 """Import datasets into this repository that were exported from a
1541 different butler repository via `~lsst.daf.butler.Butler.export`.
1543 Parameters
1544 ----------
1545 directory : `~lsst.resources.ResourcePathExpression`, optional
1546 Directory containing dataset files to import from. If `None`,
1547 ``filename`` and all dataset file paths specified therein must
1548 be absolute.
1549 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1550 A stream or name of file that contains database information
1551 associated with the exported datasets, typically generated by
1552 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1553 `~lsst.resources.ResourcePath` and is not an absolute path,
1554 it will first be looked for relative to ``directory`` and if not
1555 found there it will be looked for in the current working
1556 directory. Defaults to "export.{format}".
1557 format : `str`, optional
1558 File format for ``filename``. If `None`, the extension of
1559 ``filename`` will be used.
1560 transfer : `str`, optional
1561 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1562 skip_dimensions : `set`, optional
1563 Names of dimensions that should be skipped and not imported.
1564 record_validation_info : `bool`, optional
1565 If `True`, the default, the datastore can record validation
1566 information associated with the file. If `False` the datastore
1567 will not attempt to track any information such as checksums
1568 or file sizes. This can be useful if such information is tracked
1569 in an external system or if the file is to be compressed in place.
1570 It is up to the datastore whether this parameter is relevant.
1571 without_datastore : `bool`, optional
1572 If `True` only registry records will be imported and the datastore
1573 will be ignored.
1575 Raises
1576 ------
1577 TypeError
1578 Raised if the set of arguments passed is inconsistent, or if the
1579 butler is read-only.
1580 """
1581 raise NotImplementedError()
1583 @abstractmethod
1584 def transfer_dimension_records_from(
1585 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef | DataCoordinate]
1586 ) -> None:
1587 """Transfer dimension records to this Butler from another Butler.
1589 Parameters
1590 ----------
1591 source_butler : `LimitedButler` or `Butler`
1592 Butler from which the records are to be transferred. If data IDs
1593 in ``source_refs`` are not expanded then this has to be a full
1594 `Butler` whose registry will be used to expand data IDs. If the
1595 source refs contain coordinates that are used to populate other
1596 records then this will also need to be a full `Butler`.
1597 source_refs : iterable of `DatasetRef` or `DataCoordinate`
1598 Datasets or data IDs defined in the source butler whose dimension
1599 records should be transferred to this butler.
1600 """
1601 raise NotImplementedError()
1603 @abstractmethod
1604 def transfer_from(
1605 self,
1606 source_butler: LimitedButler,
1607 source_refs: Iterable[DatasetRef],
1608 transfer: str = "auto",
1609 skip_missing: bool = True,
1610 register_dataset_types: bool = False,
1611 transfer_dimensions: bool = False,
1612 dry_run: bool = False,
1613 ) -> Collection[DatasetRef]:
1614 """Transfer datasets to this Butler from a run in another Butler.
1616 Parameters
1617 ----------
1618 source_butler : `LimitedButler`
1619 Butler from which the datasets are to be transferred. If data IDs
1620 in ``source_refs`` are not expanded then this has to be a full
1621 `Butler` whose registry will be used to expand data IDs.
1622 source_refs : iterable of `DatasetRef`
1623 Datasets defined in the source butler that should be transferred to
1624 this butler. In most circumstances, ``transfer_from`` is faster if
1625 the dataset refs are expanded.
1626 transfer : `str`, optional
1627 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1628 skip_missing : `bool`
1629 If `True`, datasets with no datastore artifact associated with
1630 them are not transferred. If `False` a registry entry will be
1631 created even if no datastore record is created (and so will
1632 look equivalent to the dataset being unstored).
1633 register_dataset_types : `bool`
1634 If `True` any missing dataset types are registered. Otherwise
1635 an exception is raised.
1636 transfer_dimensions : `bool`, optional
1637 If `True`, dimension record data associated with the new datasets
1638 will be transferred.
1639 dry_run : `bool`, optional
1640 If `True` the transfer will be processed without any modifications
1641 made to the target butler and as if the target butler did not
1642 have any of the datasets.
1644 Returns
1645 -------
1646 refs : `list` of `DatasetRef`
1647 The refs added to this Butler.
1649 Notes
1650 -----
1651 The datastore artifact has to exist for a transfer
1652 to be made but non-existence is not an error.
1654 Datasets that already exist in this run will be skipped.
1656 The datasets are imported as part of a transaction, although
1657 dataset types are registered before the transaction is started.
1658 This means that it is possible for a dataset type to be registered
1659 even though transfer has failed.
1660 """
1661 raise NotImplementedError()
1663 @abstractmethod
1664 def validateConfiguration(
1665 self,
1666 logFailures: bool = False,
1667 datasetTypeNames: Iterable[str] | None = None,
1668 ignore: Iterable[str] | None = None,
1669 ) -> None:
1670 """Validate butler configuration.
1672 Checks that each `DatasetType` can be stored in the `Datastore`.
1674 Parameters
1675 ----------
1676 logFailures : `bool`, optional
1677 If `True`, output a log message for every validation error
1678 detected.
1679 datasetTypeNames : iterable of `str`, optional
1680 The `DatasetType` names that should be checked. This allows
1681 only a subset to be selected.
1682 ignore : iterable of `str`, optional
1683 Names of DatasetTypes to skip over. This can be used to skip
1684 known problems. If a named `DatasetType` corresponds to a
1685 composite, all components of that `DatasetType` will also be
1686 ignored.
1688 Raises
1689 ------
1690 ButlerValidationError
1691 Raised if there is some inconsistency with how this Butler
1692 is configured.
1693 """
1694 raise NotImplementedError()
1696 @property
1697 @abstractmethod
1698 def collection_chains(self) -> ButlerCollections:
1699 """Object with methods for modifying collection chains
1700 (`~lsst.daf.butler.ButlerCollections`).
1702 Deprecated. Replaced with ``collections`` property.
1703 """
1704 raise NotImplementedError()
1706 @property
1707 @abstractmethod
1708 def collections(self) -> ButlerCollections:
1709 """Object with methods for modifying and querying collections
1710 (`~lsst.daf.butler.ButlerCollections`).
1712 Use of this object is preferred over `registry` wherever possible.
1713 """
1714 raise NotImplementedError()
1716 @property
1717 @abstractmethod
1718 def run(self) -> str | None:
1719 """Name of the run this butler writes outputs to by default (`str` or
1720 `None`).
1721 """
1722 raise NotImplementedError()
1724 @property
1725 @abstractmethod
1726 def registry(self) -> Registry:
1727 """The object that manages dataset metadata and relationships
1728 (`Registry`).
1730 Many operations that don't involve reading or writing butler datasets
1731 are accessible only via `Registry` methods. Eventually these methods
1732 will be replaced by equivalent `Butler` methods.
1733 """
1734 raise NotImplementedError()
1736 @abstractmethod
1737 def query(self) -> AbstractContextManager[Query]:
1738 """Context manager returning a `Query` object used for construction
1739 and execution of complex queries.
1740 """
1741 raise NotImplementedError()
1743 def query_data_ids(
1744 self,
1745 dimensions: DimensionGroup | Iterable[str] | str,
1746 *,
1747 data_id: DataId | None = None,
1748 where: str = "",
1749 bind: Mapping[str, Any] | None = None,
1750 with_dimension_records: bool = False,
1751 order_by: Iterable[str] | str | None = None,
1752 limit: int | None = -20_000,
1753 explain: bool = True,
1754 **kwargs: Any,
1755 ) -> list[DataCoordinate]:
1756 """Query for data IDs matching user-provided criteria.
1758 Parameters
1759 ----------
1760 dimensions : `DimensionGroup`, `str`, or \
1761 `~collections.abc.Iterable` [`str`]
1762 The dimensions of the data IDs to yield, as either `DimensionGroup`
1763 instances or `str`. Will be automatically expanded to a complete
1764 `DimensionGroup`.
1765 data_id : `dict` or `DataCoordinate`, optional
1766 A data ID whose key-value pairs are used as equality constraints
1767 in the query.
1768 where : `str`, optional
1769 A string expression similar to a SQL WHERE clause. May involve
1770 any column of a dimension table or (as a shortcut for the primary
1771 key column of a dimension table) dimension name. See
1772 :ref:`daf_butler_dimension_expressions` for more information.
1773 bind : `~collections.abc.Mapping`, optional
1774 Mapping containing literal values that should be injected into the
1775 ``where`` expression, keyed by the identifiers they replace.
1776 Values of collection type can be expanded in some cases; see
1777 :ref:`daf_butler_dimension_expressions_identifiers` for more
1778 information.
1779 with_dimension_records : `bool`, optional
1780 If `True` (default is `False`) then returned data IDs will have
1781 dimension records.
1782 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1783 Names of the columns/dimensions to use for ordering returned data
1784 IDs. Column name can be prefixed with minus (``-``) to use
1785 descending ordering.
1786 limit : `int` or `None`, optional
1787 Upper limit on the number of returned records. `None` can be used
1788 if no limit is wanted. A limit of ``0`` means that the query will
1789 be executed and validated but no results will be returned. In this
1790 case there will be no exception even if ``explain`` is `True`.
1791 If a negative value is given a warning will be issued if the number
1792 of results is capped by that limit.
1793 explain : `bool`, optional
1794 If `True` (default) then `EmptyQueryResultError` exception is
1795 raised when resulting list is empty. The exception contains
1796 non-empty list of strings explaining possible causes for empty
1797 result.
1798 **kwargs
1799 Additional keyword arguments are forwarded to
1800 `DataCoordinate.standardize` when processing the ``data_id``
1801 argument (and may be used to provide a constraining data ID even
1802 when the ``data_id`` argument is `None`).
1804 Returns
1805 -------
1806 dataIds : `list` [`DataCoordinate`]
1807 Data IDs matching the given query parameters. These are always
1808 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1809 returns `True`).
1811 Raises
1812 ------
1813 lsst.daf.butler.registry.DataIdError
1814 Raised when ``data_id`` or keyword arguments specify unknown
1815 dimensions or values, or when they contain inconsistent values.
1816 lsst.daf.butler.registry.UserExpressionError
1817 Raised when ``where`` expression is invalid.
1818 lsst.daf.butler.EmptyQueryResultError
1819 Raised when query generates empty result and ``explain`` is set to
1820 `True`.
1821 TypeError
1822 Raised when the arguments are incompatible.
1823 """
1824 if data_id is None:
1825 data_id = DataCoordinate.make_empty(self.dimensions)
1826 if order_by is None:
1827 order_by = []
1828 query_limit = limit
1829 warn_limit = False
1830 if limit is not None and limit < 0:
1831 query_limit = abs(limit) + 1
1832 warn_limit = True
1833 with self.query() as query:
1834 result = (
1835 query.data_ids(dimensions)
1836 .where(data_id, where, bind=bind, **kwargs)
1837 .order_by(*ensure_iterable(order_by))
1838 .limit(query_limit)
1839 )
1840 if with_dimension_records:
1841 result = result.with_dimension_records()
1842 data_ids = list(result)
1843 if warn_limit and len(data_ids) == query_limit:
1844 # We asked for one too many so must remove that from the list.
1845 data_ids.pop(-1)
1846 assert limit is not None # For mypy.
1847 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1848 if explain and (limit is None or limit != 0) and not data_ids:
1849 raise EmptyQueryResultError(list(result.explain_no_results()))
1850 return data_ids
1852 def query_datasets(
1853 self,
1854 dataset_type: str | DatasetType,
1855 collections: str | Iterable[str] | None = None,
1856 *,
1857 find_first: bool = True,
1858 data_id: DataId | None = None,
1859 where: str = "",
1860 bind: Mapping[str, Any] | None = None,
1861 with_dimension_records: bool = False,
1862 order_by: Iterable[str] | str | None = None,
1863 limit: int | None = -20_000,
1864 explain: bool = True,
1865 **kwargs: Any,
1866 ) -> list[DatasetRef]:
1867 """Query for dataset references matching user-provided criteria.
1869 Parameters
1870 ----------
1871 dataset_type : `str` or `DatasetType`
1872 Dataset type object or name to search for.
1873 collections : collection expression, optional
1874 A collection name or iterable of collection names to search. If not
1875 provided, the default collections are used. Can be a wildcard if
1876 ``find_first`` is `False` (if find first is requested the order
1877 of collections matters and wildcards make the order indeterminate).
1878 See :ref:`daf_butler_collection_expressions` for more information.
1879 find_first : `bool`, optional
1880 If `True` (default), for each result data ID, only yield one
1881 `DatasetRef` of each `DatasetType`, from the first collection in
1882 which a dataset of that dataset type appears (according to the
1883 order of ``collections`` passed in). If `True`, ``collections``
1884 must not contain wildcards.
1885 data_id : `dict` or `DataCoordinate`, optional
1886 A data ID whose key-value pairs are used as equality constraints in
1887 the query.
1888 where : `str`, optional
1889 A string expression similar to a SQL WHERE clause. May involve any
1890 column of a dimension table or (as a shortcut for the primary key
1891 column of a dimension table) dimension name. See
1892 :ref:`daf_butler_dimension_expressions` for more information.
1893 bind : `~collections.abc.Mapping`, optional
1894 Mapping containing literal values that should be injected into the
1895 ``where`` expression, keyed by the identifiers they replace. Values
1896 of collection type can be expanded in some cases; see
1897 :ref:`daf_butler_dimension_expressions_identifiers` for more
1898 information.
1899 with_dimension_records : `bool`, optional
1900 If `True` (default is `False`) then returned data IDs will have
1901 dimension records.
1902 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1903 Names of the columns/dimensions to use for ordering returned data
1904 IDs. Column name can be prefixed with minus (``-``) to use
1905 descending ordering.
1906 limit : `int` or `None`, optional
1907 Upper limit on the number of returned records. `None` can be used
1908 if no limit is wanted. A limit of ``0`` means that the query will
1909 be executed and validated but no results will be returned. In this
1910 case there will be no exception even if ``explain`` is `True`.
1911 If a negative value is given a warning will be issued if the number
1912 of results is capped by that limit.
1913 explain : `bool`, optional
1914 If `True` (default) then `EmptyQueryResultError` exception is
1915 raised when resulting list is empty. The exception contains
1916 non-empty list of strings explaining possible causes for empty
1917 result.
1918 **kwargs
1919 Additional keyword arguments are forwarded to
1920 `DataCoordinate.standardize` when processing the ``data_id``
1921 argument (and may be used to provide a constraining data ID even
1922 when the ``data_id`` argument is `None`).
1924 Returns
1925 -------
1926 refs : `.queries.DatasetRefQueryResults`
1927 Dataset references matching the given query criteria. Nested data
1928 IDs are guaranteed to include values for all implied dimensions
1929 (i.e. `DataCoordinate.hasFull` will return `True`).
1931 Raises
1932 ------
1933 lsst.daf.butler.registry.DatasetTypeExpressionError
1934 Raised when ``dataset_type`` expression is invalid.
1935 lsst.daf.butler.registry.DataIdError
1936 Raised when ``data_id`` or keyword arguments specify unknown
1937 dimensions or values, or when they contain inconsistent values.
1938 lsst.daf.butler.registry.UserExpressionError
1939 Raised when ``where`` expression is invalid.
1940 lsst.daf.butler.EmptyQueryResultError
1941 Raised when query generates empty result and ``explain`` is set to
1942 `True`.
1943 TypeError
1944 Raised when the arguments are incompatible, such as when a
1945 collection wildcard is passed when ``find_first`` is `True`, or
1946 when ``collections`` is `None` and default butler collections are
1947 not defined.
1948 """
1949 if data_id is None:
1950 data_id = DataCoordinate.make_empty(self.dimensions)
1951 if order_by is None:
1952 order_by = []
1953 if collections and has_globs(collections):
1954 # Wild cards need to be expanded but can only be allowed if
1955 # find_first=False because expanding wildcards does not return
1956 # a guaranteed ordering. Querying collection registry to expand
1957 # collections when we do not have wildcards is expensive so only
1958 # do it if we need it.
1959 if find_first:
1960 raise InvalidQueryError(
1961 f"Can not use wildcards in collections when find_first=True (given {collections})"
1962 )
1963 collections = self.collections.query(collections)
1964 query_limit = limit
1965 warn_limit = False
1966 if limit is not None and limit < 0:
1967 query_limit = abs(limit) + 1
1968 warn_limit = True
1969 with self.query() as query:
1970 result = (
1971 query.datasets(dataset_type, collections=collections, find_first=find_first)
1972 .where(data_id, where, bind=bind, **kwargs)
1973 .order_by(*ensure_iterable(order_by))
1974 .limit(query_limit)
1975 )
1976 if with_dimension_records:
1977 result = result.with_dimension_records()
1978 refs = list(result)
1979 if warn_limit and len(refs) == query_limit:
1980 # We asked for one too many so must remove that from the list.
1981 refs.pop(-1)
1982 assert limit is not None # For mypy.
1983 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1984 if explain and (limit is None or limit != 0) and not refs:
1985 raise EmptyQueryResultError(list(result.explain_no_results()))
1986 return refs
1988 def query_dimension_records(
1989 self,
1990 element: str,
1991 *,
1992 data_id: DataId | None = None,
1993 where: str = "",
1994 bind: Mapping[str, Any] | None = None,
1995 order_by: Iterable[str] | str | None = None,
1996 limit: int | None = -20_000,
1997 explain: bool = True,
1998 **kwargs: Any,
1999 ) -> list[DimensionRecord]:
2000 """Query for dimension information matching user-provided criteria.
2002 Parameters
2003 ----------
2004 element : `str`
2005 The name of a dimension element to obtain records for.
2006 data_id : `dict` or `DataCoordinate`, optional
2007 A data ID whose key-value pairs are used as equality constraints
2008 in the query.
2009 where : `str`, optional
2010 A string expression similar to a SQL WHERE clause. See
2011 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
2012 information.
2013 bind : `~collections.abc.Mapping`, optional
2014 Mapping containing literal values that should be injected into the
2015 ``where`` expression, keyed by the identifiers they replace.
2016 Values of collection type can be expanded in some cases; see
2017 :ref:`daf_butler_dimension_expressions_identifiers` for more
2018 information.
2019 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2020 Names of the columns/dimensions to use for ordering returned data
2021 IDs. Column name can be prefixed with minus (``-``) to use
2022 descending ordering.
2023 limit : `int` or `None`, optional
2024 Upper limit on the number of returned records. `None` can be used
2025 if no limit is wanted. A limit of ``0`` means that the query will
2026 be executed and validated but no results will be returned. In this
2027 case there will be no exception even if ``explain`` is `True`.
2028 If a negative value is given a warning will be issued if the number
2029 of results is capped by that limit.
2030 explain : `bool`, optional
2031 If `True` (default) then `EmptyQueryResultError` exception is
2032 raised when resulting list is empty. The exception contains
2033 non-empty list of strings explaining possible causes for empty
2034 result.
2035 **kwargs
2036 Additional keyword arguments are forwarded to
2037 `DataCoordinate.standardize` when processing the ``data_id``
2038 argument (and may be used to provide a constraining data ID even
2039 when the ``data_id`` argument is `None`).
2041 Returns
2042 -------
2043 records : `list` [`DimensionRecord`]
2044 Dimension records matching the given query parameters.
2046 Raises
2047 ------
2048 lsst.daf.butler.registry.DataIdError
2049 Raised when ``data_id`` or keyword arguments specify unknown
2050 dimensions or values, or when they contain inconsistent values.
2051 lsst.daf.butler.registry.UserExpressionError
2052 Raised when ``where`` expression is invalid.
2053 lsst.daf.butler.EmptyQueryResultError
2054 Raised when query generates empty result and ``explain`` is set to
2055 `True`.
2056 TypeError
2057 Raised when the arguments are incompatible, such as when a
2058 collection wildcard is passed when ``find_first`` is `True`, or
2059 when ``collections`` is `None` and default butler collections are
2060 not defined.
2061 """
2062 if data_id is None:
2063 data_id = DataCoordinate.make_empty(self.dimensions)
2064 if order_by is None:
2065 order_by = []
2066 query_limit = limit
2067 warn_limit = False
2068 if limit is not None and limit < 0:
2069 query_limit = abs(limit) + 1
2070 warn_limit = True
2071 with self.query() as query:
2072 result = (
2073 query.dimension_records(element)
2074 .where(data_id, where, bind=bind, **kwargs)
2075 .order_by(*ensure_iterable(order_by))
2076 .limit(query_limit)
2077 )
2078 dimension_records = list(result)
2079 if warn_limit and len(dimension_records) == query_limit:
2080 # We asked for one too many so must remove that from the list.
2081 dimension_records.pop(-1)
2082 assert limit is not None # For mypy.
2083 _LOG.warning(
2084 "More dimension records are available than the requested limit of %d.", abs(limit)
2085 )
2086 if explain and (limit is None or limit != 0) and not dimension_records:
2087 raise EmptyQueryResultError(list(result.explain_no_results()))
2088 return dimension_records
2090 def query_all_datasets(
2091 self,
2092 collections: str | Iterable[str] | None = None,
2093 *,
2094 name: str | Iterable[str] = "*",
2095 find_first: bool = True,
2096 data_id: DataId | None = None,
2097 where: str = "",
2098 bind: Mapping[str, Any] | None = None,
2099 limit: int | None = -20_000,
2100 **kwargs: Any,
2101 ) -> list[DatasetRef]:
2102 """Query for datasets of potentially multiple types.
2104 Parameters
2105 ----------
2106 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2107 The collection or collections to search, in order. If not provided
2108 or `None`, the default collection search path for this butler is
2109 used.
2110 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2111 Names or name patterns (glob-style) that returned dataset type
2112 names must match. If an iterable, items are OR'd together. The
2113 default is to include all dataset types in the given collections.
2114 find_first : `bool`, optional
2115 If `True` (default), for each result data ID, only yield one
2116 `DatasetRef` of each `DatasetType`, from the first collection in
2117 which a dataset of that dataset type appears (according to the
2118 order of ``collections`` passed in).
2119 data_id : `dict` or `DataCoordinate`, optional
2120 A data ID whose key-value pairs are used as equality constraints in
2121 the query.
2122 where : `str`, optional
2123 A string expression similar to a SQL WHERE clause. May involve any
2124 column of a dimension table or (as a shortcut for the primary key
2125 column of a dimension table) dimension name. See
2126 :ref:`daf_butler_dimension_expressions` for more information.
2127 bind : `~collections.abc.Mapping`, optional
2128 Mapping containing literal values that should be injected into the
2129 ``where`` expression, keyed by the identifiers they replace. Values
2130 of collection type can be expanded in some cases; see
2131 :ref:`daf_butler_dimension_expressions_identifiers` for more
2132 information.
2133 limit : `int` or `None`, optional
2134 Upper limit on the number of returned records. `None` can be used
2135 if no limit is wanted. A limit of ``0`` means that the query will
2136 be executed and validated but no results will be returned.
2137 If a negative value is given a warning will be issued if the number
2138 of results is capped by that limit. If no limit is provided, by
2139 default a maximum of 20,000 records will be returned.
2140 **kwargs
2141 Additional keyword arguments are forwarded to
2142 `DataCoordinate.standardize` when processing the ``data_id``
2143 argument (and may be used to provide a constraining data ID even
2144 when the ``data_id`` argument is `None`).
2146 Raises
2147 ------
2148 MissingDatasetTypeError
2149 When no dataset types match ``name``, or an explicit (non-glob)
2150 dataset type in ``name`` does not exist.
2151 InvalidQueryError
2152 If the parameters to the query are inconsistent or malformed.
2153 MissingCollectionError
2154 If a given collection is not found.
2156 Returns
2157 -------
2158 refs : `list` [ `DatasetRef` ]
2159 Dataset references matching the given query criteria. Nested data
2160 IDs are guaranteed to include values for all implied dimensions
2161 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2162 include dimension records (`DataCoordinate.hasRecords` will be
2163 `False`).
2164 """
2165 if collections is None:
2166 collections = list(self.collections.defaults)
2167 else:
2168 collections = list(ensure_iterable(collections))
2170 if bind is None:
2171 bind = {}
2172 if data_id is None:
2173 data_id = {}
2175 warn_limit = False
2176 if limit is not None and limit < 0:
2177 # Add one to the limit so we can detect if we have exceeded it.
2178 limit = abs(limit) + 1
2179 warn_limit = True
2181 args = QueryAllDatasetsParameters(
2182 collections=collections,
2183 name=list(ensure_iterable(name)),
2184 find_first=find_first,
2185 data_id=data_id,
2186 where=where,
2187 limit=limit,
2188 bind=bind,
2189 kwargs=kwargs,
2190 with_dimension_records=False,
2191 )
2192 with self._query_all_datasets_by_page(args) as pages:
2193 result = []
2194 for page in pages:
2195 result.extend(page)
2197 if warn_limit and limit is not None and len(result) >= limit:
2198 # Remove the extra dataset we added for the limit check.
2199 result.pop()
2200 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2202 return result
2204 @abstractmethod
2205 def _query_all_datasets_by_page(
2206 self, args: QueryAllDatasetsParameters
2207 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2208 raise NotImplementedError()
2210 def clone(
2211 self,
2212 *,
2213 collections: CollectionArgType | None | EllipsisType = ...,
2214 run: str | None | EllipsisType = ...,
2215 inferDefaults: bool | EllipsisType = ...,
2216 dataId: dict[str, str] | EllipsisType = ...,
2217 metrics: ButlerMetrics | None = None,
2218 ) -> Butler:
2219 """Return a new Butler instance connected to the same repository
2220 as this one, optionally overriding ``collections``, ``run``,
2221 ``inferDefaults``, and default data ID.
2223 Parameters
2224 ----------
2225 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2226 optional
2227 Same as constructor. If omitted, uses value from original object.
2228 run : `str` or `None`, optional
2229 Same as constructor. If `None`, no default run is used. If
2230 omitted, copies value from original object.
2231 inferDefaults : `bool`, optional
2232 Same as constructor. If omitted, copies value from original
2233 object.
2234 dataId : `str`
2235 Same as ``kwargs`` passed to the constructor. If omitted, copies
2236 values from original object.
2237 metrics : `ButlerMetrics` or `None`, optional
2238 Metrics object to record butler statistics.
2239 """
2240 raise NotImplementedError()
2242 @abstractmethod
2243 def close(self) -> None:
2244 raise NotImplementedError()
2246 @abstractmethod
2247 def _expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
2248 raise NotImplementedError()