Coverage for python / lsst / daf / butler / _butler.py: 30%
308 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-07 01:18 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 **kwargs : `Any`
142 Additional keyword arguments passed to a constructor of actual butler
143 class.
145 Notes
146 -----
147 The preferred way to instantiate Butler is via the `from_config` method.
148 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
149 but ``mypy`` will complain about the former.
150 """
152 def __new__(
153 cls,
154 config: Config | ResourcePathExpression | None = None,
155 *,
156 collections: Any = None,
157 run: str | None = None,
158 searchPaths: Sequence[ResourcePathExpression] | None = None,
159 writeable: bool | None = None,
160 inferDefaults: bool = True,
161 without_datastore: bool = False,
162 metrics: ButlerMetrics | None = None,
163 **kwargs: Any,
164 ) -> Butler:
165 if cls is Butler:
166 return Butler.from_config(
167 config=config,
168 collections=collections,
169 run=run,
170 searchPaths=searchPaths,
171 writeable=writeable,
172 inferDefaults=inferDefaults,
173 without_datastore=without_datastore,
174 metrics=metrics,
175 **kwargs,
176 )
178 # Note: we do not pass any parameters to __new__, Python will pass them
179 # to __init__ after __new__ returns sub-class instance.
180 return super().__new__(cls)
182 @classmethod
183 def from_config(
184 cls,
185 config: Config | ResourcePathExpression | None = None,
186 *,
187 collections: Any = None,
188 run: str | None = None,
189 searchPaths: Sequence[ResourcePathExpression] | None = None,
190 writeable: bool | None = None,
191 inferDefaults: bool = True,
192 without_datastore: bool = False,
193 metrics: ButlerMetrics | None = None,
194 **kwargs: Any,
195 ) -> Butler:
196 """Create butler instance from configuration.
198 Parameters
199 ----------
200 config : `ButlerConfig`, `Config` or `str`, optional
201 Configuration. Anything acceptable to the `ButlerConfig`
202 constructor. If a directory path is given the configuration will be
203 read from a ``butler.yaml`` file in that location. If `None` is
204 given default values will be used. If ``config`` contains "cls" key
205 then its value is used as a name of butler class and it must be a
206 sub-class of this class, otherwise `DirectButler` is instantiated.
207 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
208 An expression specifying the collections to be searched (in order)
209 when reading datasets.
210 This may be a `str` collection name or an iterable thereof.
211 See :ref:`daf_butler_collection_expressions` for more information.
212 These collections are not registered automatically and must be
213 manually registered before they are used by any method, but they
214 may be manually registered after the `Butler` is initialized.
215 run : `str`, optional
216 Name of the `~CollectionType.RUN` collection new datasets should be
217 inserted into. If ``collections`` is `None` and ``run`` is not
218 `None`, ``collections`` will be set to ``[run]``. If not `None`,
219 this collection will automatically be registered. If this is not
220 set (and ``writeable`` is not set either), a read-only butler will
221 be created.
222 searchPaths : `list` of `str`, optional
223 Directory paths to search when calculating the full Butler
224 configuration. Not used if the supplied config is already a
225 `ButlerConfig`.
226 writeable : `bool`, optional
227 Explicitly sets whether the butler supports write operations. If
228 not provided, a read-write butler is created if any of ``run``,
229 ``tags``, or ``chains`` is non-empty.
230 inferDefaults : `bool`, optional
231 If `True` (default) infer default data ID values from the values
232 present in the datasets in ``collections``: if all collections have
233 the same value (or no value) for a governor dimension, that value
234 will be the default for that dimension. Nonexistent collections
235 are ignored. If a default value is provided explicitly for a
236 governor dimension via ``**kwargs``, no default will be inferred
237 for that dimension.
238 without_datastore : `bool`, optional
239 If `True` do not attach a datastore to this butler. Any attempts
240 to use a datastore will fail.
241 metrics : `ButlerMetrics` or `None`, optional
242 Metrics object to record butler usage statistics.
243 **kwargs : `Any`
244 Default data ID key-value pairs. These may only identify
245 "governor" dimensions like ``instrument`` and ``skymap``.
247 Returns
248 -------
249 butler : `Butler`
250 A `Butler` constructed from the given configuration.
252 Notes
253 -----
254 Calling this factory method is identical to calling
255 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
256 complains about ``Butler()`` call.
258 Examples
259 --------
260 While there are many ways to control exactly how a `Butler` interacts
261 with the collections in its `Registry`, the most common cases are still
262 simple.
264 For a read-only `Butler` that searches one collection, do::
266 butler = Butler.from_config(
267 "/path/to/repo", collections=["u/alice/DM-50000"]
268 )
270 For a read-write `Butler` that writes to and reads from a
271 `~CollectionType.RUN` collection::
273 butler = Butler.from_config(
274 "/path/to/repo", run="u/alice/DM-50000/a"
275 )
277 The `Butler` passed to a ``PipelineTask`` is often much more complex,
278 because we want to write to one `~CollectionType.RUN` collection but
279 read from several others (as well)::
281 butler = Butler.from_config(
282 "/path/to/repo",
283 run="u/alice/DM-50000/a",
284 collections=[
285 "u/alice/DM-50000/a",
286 "u/bob/DM-49998",
287 "HSC/defaults",
288 ],
289 )
291 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
292 Datasets will be read first from that run (since it appears first in
293 the chain), and then from ``u/bob/DM-49998`` and finally
294 ``HSC/defaults``.
296 Finally, one can always create a `Butler` with no collections::
298 butler = Butler.from_config("/path/to/repo", writeable=True)
300 This can be extremely useful when you just want to use
301 ``butler.registry``, e.g. for inserting dimension data or managing
302 collections, or when the collections you want to use with the butler
303 are not consistent. Passing ``writeable`` explicitly here is only
304 necessary if you want to be able to make changes to the repo - usually
305 the value for ``writeable`` can be guessed from the collection
306 arguments provided, but it defaults to `False` when there are not
307 collection arguments.
308 """
309 # DirectButler used to have a way to specify a "copy constructor" by
310 # passing the "butler" parameter to its constructor. This has
311 # been moved out of the constructor into Butler.clone().
312 butler = kwargs.pop("butler", None)
313 metrics = metrics if metrics is not None else ButlerMetrics()
314 if butler is not None:
315 if not isinstance(butler, Butler):
316 raise TypeError("'butler' parameter must be a Butler instance")
317 if config is not None or searchPaths is not None or writeable is not None:
318 raise TypeError(
319 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
320 )
321 return butler.clone(
322 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
323 )
325 options = ButlerInstanceOptions(
326 collections=collections,
327 run=run,
328 writeable=writeable,
329 inferDefaults=inferDefaults,
330 metrics=metrics,
331 kwargs=kwargs,
332 )
334 # Load the Butler configuration. This may involve searching the
335 # environment to locate a configuration file.
336 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
337 butler_type = butler_config.get_butler_type()
339 # Make DirectButler if class is not specified.
340 match butler_type:
341 case ButlerType.DIRECT:
342 from .direct_butler import DirectButler
344 return DirectButler.create_from_config(
345 butler_config,
346 options=options,
347 without_datastore=without_datastore,
348 )
349 case ButlerType.REMOTE:
350 from .remote_butler._factory import RemoteButlerFactory
352 # Assume this is being created by a client who would like
353 # default caching of remote datasets.
354 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
355 return factory.create_butler_with_credentials_from_environment(
356 butler_options=options, enable_datastore_cache=True
357 )
358 case _:
359 raise TypeError(f"Unknown Butler type '{butler_type}'")
361 @staticmethod
362 def makeRepo(
363 root: ResourcePathExpression,
364 config: Config | str | None = None,
365 dimensionConfig: Config | str | None = None,
366 standalone: bool = False,
367 searchPaths: list[str] | None = None,
368 forceConfigRoot: bool = True,
369 outfile: ResourcePathExpression | None = None,
370 overwrite: bool = False,
371 ) -> Config:
372 """Create an empty data repository by adding a butler.yaml config
373 to a repository root directory.
375 Parameters
376 ----------
377 root : `lsst.resources.ResourcePathExpression`
378 Path or URI to the root location of the new repository. Will be
379 created if it does not exist.
380 config : `Config` or `str`, optional
381 Configuration to write to the repository, after setting any
382 root-dependent Registry or Datastore config options. Can not
383 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
384 configuration will be used. Root-dependent config options
385 specified in this config are overwritten if ``forceConfigRoot``
386 is `True`.
387 dimensionConfig : `Config` or `str`, optional
388 Configuration for dimensions, will be used to initialize registry
389 database.
390 standalone : `bool`
391 If True, write all expanded defaults, not just customized or
392 repository-specific settings.
393 This (mostly) decouples the repository from the default
394 configuration, insulating it from changes to the defaults (which
395 may be good or bad, depending on the nature of the changes).
396 Future *additions* to the defaults will still be picked up when
397 initializing `Butlers` to repos created with ``standalone=True``.
398 searchPaths : `list` of `str`, optional
399 Directory paths to search when calculating the full butler
400 configuration.
401 forceConfigRoot : `bool`, optional
402 If `False`, any values present in the supplied ``config`` that
403 would normally be reset are not overridden and will appear
404 directly in the output config. This allows non-standard overrides
405 of the root directory for a datastore or registry to be given.
406 If this parameter is `True` the values for ``root`` will be
407 forced into the resulting config if appropriate.
408 outfile : `lss.resources.ResourcePathExpression`, optional
409 If not-`None`, the output configuration will be written to this
410 location rather than into the repository itself. Can be a URI
411 string. Can refer to a directory that will be used to write
412 ``butler.yaml``.
413 overwrite : `bool`, optional
414 Create a new configuration file even if one already exists
415 in the specified output location. Default is to raise
416 an exception.
418 Returns
419 -------
420 config : `Config`
421 The updated `Config` instance written to the repo.
423 Raises
424 ------
425 ValueError
426 Raised if a ButlerConfig or ConfigSubset is passed instead of a
427 regular Config (as these subclasses would make it impossible to
428 support ``standalone=False``).
429 FileExistsError
430 Raised if the output config file already exists.
431 os.error
432 Raised if the directory does not exist, exists but is not a
433 directory, or cannot be created.
435 Notes
436 -----
437 Note that when ``standalone=False`` (the default), the configuration
438 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
439 construct the repository should also be used to construct any Butlers
440 to avoid configuration inconsistencies.
441 """
442 if isinstance(config, ButlerConfig | ConfigSubset):
443 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
445 # Ensure that the root of the repository exists or can be made
446 root_uri = ResourcePath(root, forceDirectory=True)
447 root_uri.mkdir()
449 config = Config(config)
451 # If we are creating a new repo from scratch with relative roots,
452 # do not propagate an explicit root from the config file
453 if "root" in config:
454 del config["root"]
456 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
457 imported_class = doImportType(full["datastore", "cls"])
458 if not issubclass(imported_class, Datastore):
459 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
460 datastoreClass: type[Datastore] = imported_class
461 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
463 # if key exists in given config, parse it, otherwise parse the defaults
464 # in the expanded config
465 if config.get(("registry", "db")):
466 registryConfig = RegistryConfig(config)
467 else:
468 registryConfig = RegistryConfig(full)
469 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
470 if defaultDatabaseUri is not None:
471 Config.updateParameters(
472 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
473 )
474 else:
475 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
477 if standalone:
478 config.merge(full)
479 else:
480 # Always expand the registry.managers section into the per-repo
481 # config, because after the database schema is created, it's not
482 # allowed to change anymore. Note that in the standalone=True
483 # branch, _everything_ in the config is expanded, so there's no
484 # need to special case this.
485 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
486 configURI: ResourcePathExpression
487 if outfile is not None:
488 # When writing to a separate location we must include
489 # the root of the butler repo in the config else it won't know
490 # where to look.
491 config["root"] = root_uri.geturl()
492 configURI = outfile
493 else:
494 configURI = root_uri
495 # Strip obscore configuration, if it is present, before writing config
496 # to a file, obscore config will be stored in registry.
497 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
498 config_to_write = config.copy()
499 del config_to_write[obscore_config_key]
500 config_to_write.dumpToUri(configURI, overwrite=overwrite)
501 # configFile attribute is updated, need to copy it to original.
502 config.configFile = config_to_write.configFile
503 else:
504 config.dumpToUri(configURI, overwrite=overwrite)
506 # Create Registry and populate tables
507 registryConfig = RegistryConfig(config.get("registry"))
508 dimensionConfig = DimensionConfig(dimensionConfig)
509 registry = _RegistryFactory(registryConfig).create_from_config(
510 dimensionConfig=dimensionConfig, butlerRoot=root_uri
511 )
512 registry.close()
514 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
516 return config
518 @classmethod
519 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
520 """Look up the label in a butler repository index.
522 Parameters
523 ----------
524 label : `str`
525 Label of the Butler repository to look up.
526 return_label : `bool`, optional
527 If ``label`` cannot be found in the repository index (either
528 because index is not defined or ``label`` is not in the index) and
529 ``return_label`` is `True` then return ``ResourcePath(label)``.
530 If ``return_label`` is `False` (default) then an exception will be
531 raised instead.
533 Returns
534 -------
535 uri : `lsst.resources.ResourcePath`
536 URI to the Butler repository associated with the given label or
537 default value if it is provided.
539 Raises
540 ------
541 KeyError
542 Raised if the label is not found in the index, or if an index
543 is not defined, and ``return_label`` is `False`.
545 Notes
546 -----
547 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
548 information is discovered.
549 """
550 return ButlerRepoIndex.get_repo_uri(label, return_label)
552 @classmethod
553 def get_known_repos(cls) -> set[str]:
554 """Retrieve the list of known repository labels.
556 Returns
557 -------
558 repos : `set` of `str`
559 All the known labels. Can be empty if no index can be found.
561 Notes
562 -----
563 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
564 information is discovered.
565 """
566 return ButlerRepoIndex.get_known_repos()
568 @classmethod
569 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
570 """Extract the butler label and dataset ID from a dataset URI.
572 Parameters
573 ----------
574 uri : `str`
575 The dataset URI to parse.
577 Returns
578 -------
579 parsed : `ParsedButlerDatasetURI`
580 The label associated with the butler repository from which this
581 dataset originates and the ID of the dataset.
583 Notes
584 -----
585 Supports dataset URIs of the forms
586 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
587 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
588 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
589 can include anything supported by the `Butler` constructor, including
590 paths to repositories and alias labels.
592 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
594 will return a label of ``/repo/main``.
596 This method does not attempt to check that the dataset exists in the
597 labeled butler.
599 Since the IVOID can be issued by any publisher to represent a Butler
600 dataset there is no validation of the path or netloc component of the
601 URI. The only requirement is that there are ``id`` and ``repo`` keys
602 in the ``ivo`` URI query component.
603 """
604 parsed = urllib.parse.urlparse(uri)
605 parsed_scheme = parsed.scheme.lower()
606 if parsed_scheme == "ivo":
607 # Do not validate the netloc or the path values.
608 qs = urllib.parse.parse_qs(parsed.query)
609 if "repo" not in qs or "id" not in qs:
610 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
611 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
612 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
613 label = qs["repo"][0]
614 id_ = qs["id"][0]
615 elif parsed_scheme == "butler":
616 label = parsed.netloc # Butler label is case sensitive.
617 # Need to strip the leading /.
618 id_ = parsed.path[1:]
619 else:
620 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
621 # Strip trailing/leading whitespace from label.
622 label = label.strip()
623 if not label:
624 raise ValueError(f"No butler repository label found in uri {uri!r}")
625 try:
626 dataset_id = uuid.UUID(hex=id_)
627 except Exception as e:
628 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
629 raise
631 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
633 @classmethod
634 def get_dataset_from_uri(
635 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
636 ) -> SpecificButlerDataset:
637 """Get the dataset associated with the given dataset URI.
639 Parameters
640 ----------
641 uri : `str`
642 The URI associated with a dataset.
643 factory : `LabeledButlerFactoryProtocol` or `None`, optional
644 Bound factory function that will be given the butler label
645 and receive a `Butler`. If this is not provided the label
646 will be tried directly.
648 Returns
649 -------
650 result : `SpecificButlerDataset`
651 The butler associated with this URI and the dataset itself.
652 The dataset can be `None` if the UUID is valid but the dataset
653 is not known to this butler.
654 """
655 parsed = cls.parse_dataset_uri(uri)
656 butler: Butler | None = None
657 if factory is not None:
658 # If the label is not recognized, it might be a path.
659 try:
660 butler = factory(parsed.label)
661 except KeyError:
662 pass
663 if butler is None:
664 butler = cls.from_config(parsed.label)
665 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
667 @abstractmethod
668 def _caching_context(self) -> AbstractContextManager[None]:
669 """Context manager that enables caching."""
670 raise NotImplementedError()
672 @abstractmethod
673 def transaction(self) -> AbstractContextManager[None]:
674 """Context manager supporting `Butler` transactions.
676 Transactions can be nested.
677 """
678 raise NotImplementedError()
680 @abstractmethod
681 def put(
682 self,
683 obj: Any,
684 datasetRefOrType: DatasetRef | DatasetType | str,
685 /,
686 dataId: DataId | None = None,
687 *,
688 run: str | None = None,
689 provenance: DatasetProvenance | None = None,
690 **kwargs: Any,
691 ) -> DatasetRef:
692 """Store and register a dataset.
694 Parameters
695 ----------
696 obj : `object`
697 The dataset.
698 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
699 When `DatasetRef` is provided, ``dataId`` should be `None`.
700 Otherwise the `DatasetType` or name thereof. If a fully resolved
701 `DatasetRef` is given the run and ID are used directly.
702 dataId : `dict` or `DataCoordinate`
703 A `dict` of `Dimension` link name, value pairs that label the
704 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
705 should be provided as the second argument.
706 run : `str`, optional
707 The name of the run the dataset should be added to, overriding
708 ``self.run``. Not used if a resolved `DatasetRef` is provided.
709 provenance : `DatasetProvenance` or `None`, optional
710 Any provenance that should be attached to the serialized dataset.
711 Not supported by all serialization mechanisms.
712 **kwargs
713 Additional keyword arguments used to augment or construct a
714 `DataCoordinate`. See `DataCoordinate.standardize`
715 parameters. Not used if a resolve `DatasetRef` is provided.
717 Returns
718 -------
719 ref : `DatasetRef`
720 A reference to the stored dataset, updated with the correct id if
721 given.
723 Raises
724 ------
725 TypeError
726 Raised if the butler is read-only or if no run has been provided.
727 """
728 raise NotImplementedError()
730 @abstractmethod
731 def getDeferred(
732 self,
733 datasetRefOrType: DatasetRef | DatasetType | str,
734 /,
735 dataId: DataId | None = None,
736 *,
737 parameters: dict | None = None,
738 collections: Any = None,
739 storageClass: str | StorageClass | None = None,
740 timespan: Timespan | None = None,
741 **kwargs: Any,
742 ) -> DeferredDatasetHandle:
743 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
744 after an immediate registry lookup.
746 Parameters
747 ----------
748 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
749 When `DatasetRef` the `dataId` should be `None`.
750 Otherwise the `DatasetType` or name thereof.
751 dataId : `dict` or `DataCoordinate`, optional
752 A `dict` of `Dimension` link name, value pairs that label the
753 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
754 should be provided as the first argument.
755 parameters : `dict`
756 Additional StorageClass-defined options to control reading,
757 typically used to efficiently read only a subset of the dataset.
758 collections : Any, optional
759 Collections to be searched, overriding ``self.collections``.
760 Can be any of the types supported by the ``collections`` argument
761 to butler construction.
762 storageClass : `StorageClass` or `str`, optional
763 The storage class to be used to override the Python type
764 returned by this method. By default the returned type matches
765 the dataset type definition for this dataset. Specifying a
766 read `StorageClass` can force a different type to be returned.
767 This type must be compatible with the original type.
768 timespan : `Timespan` or `None`, optional
769 A timespan that the validity range of the dataset must overlap.
770 If not provided and this is a calibration dataset type, an attempt
771 will be made to find the timespan from any temporal coordinate
772 in the data ID.
773 **kwargs
774 Additional keyword arguments used to augment or construct a
775 `DataId`. See `DataId` parameters.
777 Returns
778 -------
779 obj : `DeferredDatasetHandle`
780 A handle which can be used to retrieve a dataset at a later time.
782 Raises
783 ------
784 LookupError
785 Raised if no matching dataset exists in the `Registry` or
786 datastore.
787 ValueError
788 Raised if a resolved `DatasetRef` was passed as an input, but it
789 differs from the one found in the registry.
790 TypeError
791 Raised if no collections were provided.
792 """
793 raise NotImplementedError()
795 @abstractmethod
796 def get(
797 self,
798 datasetRefOrType: DatasetRef | DatasetType | str,
799 /,
800 dataId: DataId | None = None,
801 *,
802 parameters: dict[str, Any] | None = None,
803 collections: Any = None,
804 storageClass: StorageClass | str | None = None,
805 timespan: Timespan | None = None,
806 **kwargs: Any,
807 ) -> Any:
808 """Retrieve a stored dataset.
810 Parameters
811 ----------
812 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
813 When `DatasetRef` the `dataId` should be `None`.
814 Otherwise the `DatasetType` or name thereof.
815 If a resolved `DatasetRef`, the associated dataset
816 is returned directly without additional querying.
817 dataId : `dict` or `DataCoordinate`
818 A `dict` of `Dimension` link name, value pairs that label the
819 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
820 should be provided as the first argument.
821 parameters : `dict`
822 Additional StorageClass-defined options to control reading,
823 typically used to efficiently read only a subset of the dataset.
824 collections : Any, optional
825 Collections to be searched, overriding ``self.collections``.
826 Can be any of the types supported by the ``collections`` argument
827 to butler construction.
828 storageClass : `StorageClass` or `str`, optional
829 The storage class to be used to override the Python type
830 returned by this method. By default the returned type matches
831 the dataset type definition for this dataset. Specifying a
832 read `StorageClass` can force a different type to be returned.
833 This type must be compatible with the original type.
834 timespan : `Timespan` or `None`, optional
835 A timespan that the validity range of the dataset must overlap.
836 If not provided and this is a calibration dataset type, an attempt
837 will be made to find the timespan from any temporal coordinate
838 in the data ID.
839 **kwargs
840 Additional keyword arguments used to augment or construct a
841 `DataCoordinate`. See `DataCoordinate.standardize`
842 parameters.
844 Returns
845 -------
846 obj : `object`
847 The dataset.
849 Raises
850 ------
851 LookupError
852 Raised if no matching dataset exists in the `Registry`.
853 TypeError
854 Raised if no collections were provided.
856 Notes
857 -----
858 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
859 this method requires that the given data ID include temporal dimensions
860 beyond the dimensions of the dataset type itself, in order to find the
861 dataset with the appropriate validity range. For example, a "bias"
862 dataset with native dimensions ``{instrument, detector}`` could be
863 fetched with a ``{instrument, detector, exposure}`` data ID, because
864 ``exposure`` is a temporal dimension.
865 """
866 raise NotImplementedError()
868 @abstractmethod
869 def getURIs(
870 self,
871 datasetRefOrType: DatasetRef | DatasetType | str,
872 /,
873 dataId: DataId | None = None,
874 *,
875 predict: bool = False,
876 collections: Any = None,
877 run: str | None = None,
878 **kwargs: Any,
879 ) -> DatasetRefURIs:
880 """Return the URIs associated with the dataset.
882 Parameters
883 ----------
884 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
885 When `DatasetRef` the `dataId` should be `None`.
886 Otherwise the `DatasetType` or name thereof.
887 dataId : `dict` or `DataCoordinate`
888 A `dict` of `Dimension` link name, value pairs that label the
889 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
890 should be provided as the first argument.
891 predict : `bool`
892 If `True`, allow URIs to be returned of datasets that have not
893 been written.
894 collections : Any, optional
895 Collections to be searched, overriding ``self.collections``.
896 Can be any of the types supported by the ``collections`` argument
897 to butler construction.
898 run : `str`, optional
899 Run to use for predictions, overriding ``self.run``.
900 **kwargs
901 Additional keyword arguments used to augment or construct a
902 `DataCoordinate`. See `DataCoordinate.standardize`
903 parameters.
905 Returns
906 -------
907 uris : `DatasetRefURIs`
908 The URI to the primary artifact associated with this dataset (if
909 the dataset was disassembled within the datastore this may be
910 `None`), and the URIs to any components associated with the dataset
911 artifact. (can be empty if there are no components).
912 """
913 raise NotImplementedError()
915 def getURI(
916 self,
917 datasetRefOrType: DatasetRef | DatasetType | str,
918 /,
919 dataId: DataId | None = None,
920 *,
921 predict: bool = False,
922 collections: Any = None,
923 run: str | None = None,
924 **kwargs: Any,
925 ) -> ResourcePath:
926 """Return the URI to the Dataset.
928 Parameters
929 ----------
930 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
931 When `DatasetRef` the `dataId` should be `None`.
932 Otherwise the `DatasetType` or name thereof.
933 dataId : `dict` or `DataCoordinate`
934 A `dict` of `Dimension` link name, value pairs that label the
935 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
936 should be provided as the first argument.
937 predict : `bool`
938 If `True`, allow URIs to be returned of datasets that have not
939 been written.
940 collections : Any, optional
941 Collections to be searched, overriding ``self.collections``.
942 Can be any of the types supported by the ``collections`` argument
943 to butler construction.
944 run : `str`, optional
945 Run to use for predictions, overriding ``self.run``.
946 **kwargs
947 Additional keyword arguments used to augment or construct a
948 `DataCoordinate`. See `DataCoordinate.standardize`
949 parameters.
951 Returns
952 -------
953 uri : `lsst.resources.ResourcePath`
954 URI pointing to the Dataset within the datastore. If the
955 Dataset does not exist in the datastore, and if ``predict`` is
956 `True`, the URI will be a prediction and will include a URI
957 fragment "#predicted".
958 If the datastore does not have entities that relate well
959 to the concept of a URI the returned URI string will be
960 descriptive. The returned URI is not guaranteed to be obtainable.
962 Raises
963 ------
964 LookupError
965 A URI has been requested for a dataset that does not exist and
966 guessing is not allowed.
967 ValueError
968 Raised if a resolved `DatasetRef` was passed as an input, but it
969 differs from the one found in the registry.
970 TypeError
971 Raised if no collections were provided.
972 RuntimeError
973 Raised if a URI is requested for a dataset that consists of
974 multiple artifacts.
975 """
976 primary, components = self.getURIs(
977 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
978 )
980 if primary is None or components:
981 raise RuntimeError(
982 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
983 "Use Butler.getURIs() instead."
984 )
985 return primary
987 @abstractmethod
988 def get_dataset_type(self, name: str) -> DatasetType:
989 """Get the `DatasetType`.
991 Parameters
992 ----------
993 name : `str`
994 Name of the type.
996 Returns
997 -------
998 type : `DatasetType`
999 The `DatasetType` associated with the given name.
1001 Raises
1002 ------
1003 lsst.daf.butler.MissingDatasetTypeError
1004 Raised if the requested dataset type has not been registered.
1006 Notes
1007 -----
1008 This method handles component dataset types automatically, though most
1009 other operations do not.
1010 """
1011 raise NotImplementedError()
1013 @abstractmethod
1014 def get_dataset(
1015 self,
1016 id: DatasetId | str,
1017 *,
1018 storage_class: str | StorageClass | None = None,
1019 dimension_records: bool = False,
1020 datastore_records: bool = False,
1021 ) -> DatasetRef | None:
1022 """Retrieve a Dataset entry.
1024 Parameters
1025 ----------
1026 id : `DatasetId`
1027 The unique identifier for the dataset, as an instance of
1028 `uuid.UUID` or a string containing a hexadecimal number.
1029 storage_class : `str` or `StorageClass` or `None`
1030 A storage class to use when creating the returned entry. If given
1031 it must be compatible with the default storage class.
1032 dimension_records : `bool`, optional
1033 If `True` the ref will be expanded and contain dimension records.
1034 datastore_records : `bool`, optional
1035 If `True` the ref will contain associated datastore records.
1037 Returns
1038 -------
1039 ref : `DatasetRef` or `None`
1040 A ref to the Dataset, or `None` if no matching Dataset
1041 was found.
1042 """
1043 raise NotImplementedError()
1045 @abstractmethod
1046 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1047 """Retrieve a list of dataset entries.
1049 Parameters
1050 ----------
1051 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1052 The unique identifiers for the datasets, as instances of
1053 `uuid.UUID` or strings containing a hexadecimal number.
1055 Returns
1056 -------
1057 refs : `list` [ `DatasetRef` ]
1058 A list containing a `DatasetRef` for each of the given dataset IDs.
1059 If a dataset was not found, no error is thrown -- it is just not
1060 included in the list. The returned datasets are in no particular
1061 order.
1062 """
1063 raise NotImplementedError()
1065 @abstractmethod
1066 def find_dataset(
1067 self,
1068 dataset_type: DatasetType | str,
1069 data_id: DataId | None = None,
1070 *,
1071 collections: str | Sequence[str] | None = None,
1072 timespan: Timespan | None = None,
1073 storage_class: str | StorageClass | None = None,
1074 dimension_records: bool = False,
1075 datastore_records: bool = False,
1076 **kwargs: Any,
1077 ) -> DatasetRef | None:
1078 """Find a dataset given its `DatasetType` and data ID.
1080 This can be used to obtain a `DatasetRef` that permits the dataset to
1081 be read from a `Datastore`. If the dataset is a component and can not
1082 be found using the provided dataset type, a dataset ref for the parent
1083 will be returned instead but with the correct dataset type.
1085 Parameters
1086 ----------
1087 dataset_type : `DatasetType` or `str`
1088 A `DatasetType` or the name of one. If this is a `DatasetType`
1089 instance, its storage class will be respected and propagated to
1090 the output, even if it differs from the dataset type definition
1091 in the registry, as long as the storage classes are convertible.
1092 data_id : `dict` or `DataCoordinate`, optional
1093 A `dict`-like object containing the `Dimension` links that identify
1094 the dataset within a collection. If it is a `dict` the dataId
1095 can include dimension record values such as ``day_obs`` and
1096 ``seq_num`` or ``full_name`` that can be used to derive the
1097 primary dimension.
1098 collections : `str` or `list` [`str`], optional
1099 A an ordered list of collections to search for the dataset.
1100 Defaults to ``self.defaults.collections``.
1101 timespan : `Timespan`, optional
1102 A timespan that the validity range of the dataset must overlap.
1103 If not provided, any `~CollectionType.CALIBRATION` collections
1104 matched by the ``collections`` argument will not be searched.
1105 storage_class : `str` or `StorageClass` or `None`
1106 A storage class to use when creating the returned entry. If given
1107 it must be compatible with the default storage class.
1108 dimension_records : `bool`, optional
1109 If `True` the ref will be expanded and contain dimension records.
1110 datastore_records : `bool`, optional
1111 If `True` the ref will contain associated datastore records.
1112 **kwargs
1113 Additional keyword arguments passed to
1114 `DataCoordinate.standardize` to convert ``dataId`` to a true
1115 `DataCoordinate` or augment an existing one. This can also include
1116 dimension record metadata that can be used to derive a primary
1117 dimension value.
1119 Returns
1120 -------
1121 ref : `DatasetRef`
1122 A reference to the dataset, or `None` if no matching Dataset
1123 was found.
1125 Raises
1126 ------
1127 lsst.daf.butler.NoDefaultCollectionError
1128 Raised if ``collections`` is `None` and
1129 ``self.collections`` is `None`.
1130 LookupError
1131 Raised if one or more data ID keys are missing.
1132 lsst.daf.butler.MissingDatasetTypeError
1133 Raised if the dataset type does not exist.
1134 lsst.daf.butler.MissingCollectionError
1135 Raised if any of ``collections`` does not exist in the registry.
1137 Notes
1138 -----
1139 This method simply returns `None` and does not raise an exception even
1140 when the set of collections searched is intrinsically incompatible with
1141 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1142 only `~CollectionType.CALIBRATION` collections are being searched.
1143 This may make it harder to debug some lookup failures, but the behavior
1144 is intentional; we consider it more important that failed searches are
1145 reported consistently, regardless of the reason, and that adding
1146 additional collections that do not contain a match to the search path
1147 never changes the behavior.
1149 This method handles component dataset types automatically, though most
1150 other query operations do not.
1151 """
1152 raise NotImplementedError()
1154 @abstractmethod
1155 def retrieve_artifacts_zip(
1156 self,
1157 refs: Iterable[DatasetRef],
1158 destination: ResourcePathExpression,
1159 overwrite: bool = True,
1160 ) -> ResourcePath:
1161 """Retrieve artifacts from a Butler and place in ZIP file.
1163 Parameters
1164 ----------
1165 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1166 The datasets to be included in the zip file.
1167 destination : `lsst.resources.ResourcePathExpression`
1168 Directory to write the new ZIP file. This directory will
1169 also be used as a staging area for the datasets being downloaded
1170 from the datastore.
1171 overwrite : `bool`, optional
1172 If `False` the output Zip will not be written if a file of the
1173 same name is already present in ``destination``.
1175 Returns
1176 -------
1177 zip_file : `lsst.resources.ResourcePath`
1178 The path to the new ZIP file.
1180 Raises
1181 ------
1182 ValueError
1183 Raised if there are no refs to retrieve.
1184 """
1185 raise NotImplementedError()
1187 @abstractmethod
1188 def retrieveArtifacts(
1189 self,
1190 refs: Iterable[DatasetRef],
1191 destination: ResourcePathExpression,
1192 transfer: str = "auto",
1193 preserve_path: bool = True,
1194 overwrite: bool = False,
1195 ) -> list[ResourcePath]:
1196 """Retrieve the artifacts associated with the supplied refs.
1198 Parameters
1199 ----------
1200 refs : iterable of `DatasetRef`
1201 The datasets for which artifacts are to be retrieved.
1202 A single ref can result in multiple artifacts. The refs must
1203 be resolved.
1204 destination : `lsst.resources.ResourcePath` or `str`
1205 Location to write the artifacts.
1206 transfer : `str`, optional
1207 Method to use to transfer the artifacts. Must be one of the options
1208 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1209 "move" is not allowed.
1210 preserve_path : `bool`, optional
1211 If `True` the full path of the artifact within the datastore
1212 is preserved. If `False` the final file component of the path
1213 is used.
1214 overwrite : `bool`, optional
1215 If `True` allow transfers to overwrite existing files at the
1216 destination.
1218 Returns
1219 -------
1220 targets : `list` of `lsst.resources.ResourcePath`
1221 URIs of file artifacts in destination location. Order is not
1222 preserved.
1224 Notes
1225 -----
1226 For non-file datastores the artifacts written to the destination
1227 may not match the representation inside the datastore. For example
1228 a hierarchical data structure in a NoSQL database may well be stored
1229 as a JSON file.
1230 """
1231 raise NotImplementedError()
1233 @abstractmethod
1234 def exists(
1235 self,
1236 dataset_ref_or_type: DatasetRef | DatasetType | str,
1237 /,
1238 data_id: DataId | None = None,
1239 *,
1240 full_check: bool = True,
1241 collections: Any = None,
1242 **kwargs: Any,
1243 ) -> DatasetExistence:
1244 """Indicate whether a dataset is known to Butler registry and
1245 datastore.
1247 Parameters
1248 ----------
1249 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1250 When `DatasetRef` the `dataId` should be `None`.
1251 Otherwise the `DatasetType` or name thereof.
1252 data_id : `dict` or `DataCoordinate`
1253 A `dict` of `Dimension` link name, value pairs that label the
1254 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1255 should be provided as the first argument.
1256 full_check : `bool`, optional
1257 If `True`, a check will be made for the actual existence of a
1258 dataset artifact. This will involve additional overhead due to
1259 the need to query an external system. If `False`, this check will
1260 be omitted, and the registry and datastore will solely be asked
1261 if they know about the dataset but no direct check for the
1262 artifact will be performed.
1263 collections : Any, optional
1264 Collections to be searched, overriding ``self.collections``.
1265 Can be any of the types supported by the ``collections`` argument
1266 to butler construction.
1267 **kwargs
1268 Additional keyword arguments used to augment or construct a
1269 `DataCoordinate`. See `DataCoordinate.standardize`
1270 parameters.
1272 Returns
1273 -------
1274 existence : `DatasetExistence`
1275 Object indicating whether the dataset is known to registry and
1276 datastore. Evaluates to `True` if the dataset is present and known
1277 to both.
1278 """
1279 raise NotImplementedError()
1281 @abstractmethod
1282 def _exists_many(
1283 self,
1284 refs: Iterable[DatasetRef],
1285 /,
1286 *,
1287 full_check: bool = True,
1288 ) -> dict[DatasetRef, DatasetExistence]:
1289 """Indicate whether multiple datasets are known to Butler registry and
1290 datastore.
1292 This is an experimental API that may change at any moment.
1294 Parameters
1295 ----------
1296 refs : iterable of `DatasetRef`
1297 The datasets to be checked.
1298 full_check : `bool`, optional
1299 If `True`, a check will be made for the actual existence of each
1300 dataset artifact. This will involve additional overhead due to
1301 the need to query an external system. If `False`, this check will
1302 be omitted, and the registry and datastore will solely be asked
1303 if they know about the dataset(s) but no direct check for the
1304 artifact(s) will be performed.
1306 Returns
1307 -------
1308 existence : dict of [`DatasetRef`, `DatasetExistence`]
1309 Mapping from the given dataset refs to an enum indicating the
1310 status of the dataset in registry and datastore.
1311 Each value evaluates to `True` if the dataset is present and known
1312 to both.
1313 """
1314 raise NotImplementedError()
1316 @abstractmethod
1317 def removeRuns(
1318 self,
1319 names: Iterable[str],
1320 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1321 *,
1322 unlink_from_chains: bool = False,
1323 ) -> None:
1324 """Remove one or more `~CollectionType.RUN` collections and the
1325 datasets within them.
1327 Parameters
1328 ----------
1329 names : `~collections.abc.Iterable` [ `str` ]
1330 The names of the collections to remove.
1331 unstore : `bool`, optional
1332 If `True` (default), delete datasets from all datastores in which
1333 they are present, and attempt to rollback the registry deletions if
1334 datastore deletions fail (which may not always be possible). If
1335 `False`, datastore records for these datasets are still removed,
1336 but any artifacts (e.g. files) will not be. This parameter is now
1337 deprecated and no longer has any effect. Files are always deleted
1338 from datastores unless they were ingested using full URIs.
1339 unlink_from_chains : `bool`, optional
1340 If `True` remove the RUN collection from any chains prior to
1341 removing the RUN. If `False` the removal will fail if any chains
1342 still refer to the RUN.
1344 Raises
1345 ------
1346 TypeError
1347 Raised if one or more collections are not of type
1348 `~CollectionType.RUN`.
1349 """
1350 raise NotImplementedError()
1352 @abstractmethod
1353 def ingest(
1354 self,
1355 *datasets: FileDataset,
1356 transfer: str | None = "auto",
1357 record_validation_info: bool = True,
1358 skip_existing: bool = False,
1359 ) -> None:
1360 """Store and register one or more datasets that already exist on disk.
1362 Parameters
1363 ----------
1364 *datasets : `FileDataset`
1365 Each positional argument is a struct containing information about
1366 a file to be ingested, including its URI (either absolute or
1367 relative to the datastore root, if applicable), a resolved
1368 `DatasetRef`, and optionally a formatter class or its
1369 fully-qualified string name. If a formatter is not provided, the
1370 formatter that would be used for `put` is assumed. On successful
1371 ingest all `FileDataset.formatter` attributes will be set to the
1372 formatter class used. `FileDataset.path` attributes may be modified
1373 to put paths in whatever the datastore considers a standardized
1374 form.
1375 transfer : `str`, optional
1376 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1377 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1378 transfer the file.
1379 record_validation_info : `bool`, optional
1380 If `True`, the default, the datastore can record validation
1381 information associated with the file. If `False` the datastore
1382 will not attempt to track any information such as checksums
1383 or file sizes. This can be useful if such information is tracked
1384 in an external system or if the file is to be compressed in place.
1385 It is up to the datastore whether this parameter is relevant.
1386 skip_existing : `bool`, optional
1387 If `True`, a dataset will not be ingested if a dataset with the
1388 same dataset ID already exists in the datastore.
1389 If `False` (the default), a `ConflictingDefinitionError` will be
1390 raised if any datasets with the same dataset ID already exist
1391 in the datastore.
1393 Raises
1394 ------
1395 TypeError
1396 Raised if the butler is read-only or if no run was provided.
1397 NotImplementedError
1398 Raised if the `Datastore` does not support the given transfer mode.
1399 DatasetTypeNotSupportedError
1400 Raised if one or more files to be ingested have a dataset type that
1401 is not supported by the `Datastore`..
1402 FileNotFoundError
1403 Raised if one of the given files does not exist.
1404 FileExistsError
1405 Raised if transfer is not `None` but the (internal) location the
1406 file would be moved to is already occupied.
1407 ConflictingDefinitionError
1408 Raised if a dataset already exists in the repository and
1409 ``skip_existing`` is `False`.
1411 Notes
1412 -----
1413 This operation is not fully exception safe: if a database operation
1414 fails, the given `FileDataset` instances may be only partially updated.
1416 It is atomic in terms of database operations (they will either all
1417 succeed or all fail) providing the database engine implements
1418 transactions correctly. It will attempt to be atomic in terms of
1419 filesystem operations as well, but this cannot be implemented
1420 rigorously for most datastores.
1421 """
1422 raise NotImplementedError()
1424 @abstractmethod
1425 def ingest_zip(
1426 self,
1427 zip_file: ResourcePathExpression,
1428 transfer: str = "auto",
1429 *,
1430 transfer_dimensions: bool = False,
1431 dry_run: bool = False,
1432 ) -> None:
1433 """Ingest a Zip file into this butler.
1435 The Zip file must have been created by `retrieve_artifacts_zip`.
1437 Parameters
1438 ----------
1439 zip_file : `lsst.resources.ResourcePathExpression`
1440 Path to the Zip file.
1441 transfer : `str`, optional
1442 Method to use to transfer the Zip into the datastore.
1443 transfer_dimensions : `bool`, optional
1444 If `True`, dimension record data associated with the new datasets
1445 will be transferred from the Zip file, if present.
1446 dry_run : `bool`, optional
1447 If `True` the ingest will be processed without any modifications
1448 made to the target butler and as if the target butler did not
1449 have any of the datasets.
1451 Notes
1452 -----
1453 Run collections and dataset types are created as needed.
1454 """
1455 raise NotImplementedError()
1457 @abstractmethod
1458 def export(
1459 self,
1460 *,
1461 directory: str | None = None,
1462 filename: str | None = None,
1463 format: str | None = None,
1464 transfer: str | None = None,
1465 ) -> AbstractContextManager[RepoExportContext]:
1466 """Export datasets from the repository represented by this `Butler`.
1468 This method is a context manager that returns a helper object
1469 (`RepoExportContext`) that is used to indicate what information from
1470 the repository should be exported.
1472 Parameters
1473 ----------
1474 directory : `str`, optional
1475 Directory dataset files should be written to if ``transfer`` is not
1476 `None`.
1477 filename : `str`, optional
1478 Name for the file that will include database information associated
1479 with the exported datasets. If this is not an absolute path and
1480 ``directory`` is not `None`, it will be written to ``directory``
1481 instead of the current working directory. Defaults to
1482 "export.{format}".
1483 format : `str`, optional
1484 File format for the database information file. If `None`, the
1485 extension of ``filename`` will be used.
1486 transfer : `str`, optional
1487 Transfer mode passed to `Datastore.export`.
1489 Raises
1490 ------
1491 TypeError
1492 Raised if the set of arguments passed is inconsistent.
1494 Examples
1495 --------
1496 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1497 methods are used to provide the iterables over data IDs and/or datasets
1498 to be exported::
1500 with butler.export("exports.yaml") as export:
1501 # Export all flats, but none of the dimension element rows
1502 # (i.e. data ID information) associated with them.
1503 export.saveDatasets(
1504 butler.registry.queryDatasets("flat"), elements=()
1505 )
1506 # Export all datasets that start with "deepCoadd_" and all of
1507 # their associated data ID information.
1508 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1509 """
1510 raise NotImplementedError()
1512 @abstractmethod
1513 def import_(
1514 self,
1515 *,
1516 directory: ResourcePathExpression | None = None,
1517 filename: ResourcePathExpression | TextIO | None = None,
1518 format: str | None = None,
1519 transfer: str | None = None,
1520 skip_dimensions: set | None = None,
1521 record_validation_info: bool = True,
1522 without_datastore: bool = False,
1523 ) -> None:
1524 """Import datasets into this repository that were exported from a
1525 different butler repository via `~lsst.daf.butler.Butler.export`.
1527 Parameters
1528 ----------
1529 directory : `~lsst.resources.ResourcePathExpression`, optional
1530 Directory containing dataset files to import from. If `None`,
1531 ``filename`` and all dataset file paths specified therein must
1532 be absolute.
1533 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1534 A stream or name of file that contains database information
1535 associated with the exported datasets, typically generated by
1536 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1537 `~lsst.resources.ResourcePath` and is not an absolute path,
1538 it will first be looked for relative to ``directory`` and if not
1539 found there it will be looked for in the current working
1540 directory. Defaults to "export.{format}".
1541 format : `str`, optional
1542 File format for ``filename``. If `None`, the extension of
1543 ``filename`` will be used.
1544 transfer : `str`, optional
1545 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1546 skip_dimensions : `set`, optional
1547 Names of dimensions that should be skipped and not imported.
1548 record_validation_info : `bool`, optional
1549 If `True`, the default, the datastore can record validation
1550 information associated with the file. If `False` the datastore
1551 will not attempt to track any information such as checksums
1552 or file sizes. This can be useful if such information is tracked
1553 in an external system or if the file is to be compressed in place.
1554 It is up to the datastore whether this parameter is relevant.
1555 without_datastore : `bool`, optional
1556 If `True` only registry records will be imported and the datastore
1557 will be ignored.
1559 Raises
1560 ------
1561 TypeError
1562 Raised if the set of arguments passed is inconsistent, or if the
1563 butler is read-only.
1564 """
1565 raise NotImplementedError()
1567 @abstractmethod
1568 def transfer_dimension_records_from(
1569 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef]
1570 ) -> None:
1571 """Transfer dimension records to this Butler from another Butler.
1573 Parameters
1574 ----------
1575 source_butler : `LimitedButler` or `Butler`
1576 Butler from which the records are to be transferred. If data IDs
1577 in ``source_refs`` are not expanded then this has to be a full
1578 `Butler` whose registry will be used to expand data IDs. If the
1579 source refs contain coordinates that are used to populate other
1580 records then this will also need to be a full `Butler`.
1581 source_refs : iterable of `DatasetRef`
1582 Datasets defined in the source butler whose dimension records
1583 should be transferred to this butler. In most circumstances.
1584 transfer is faster if the dataset refs are expanded.
1585 """
1586 raise NotImplementedError()
1588 @abstractmethod
1589 def transfer_from(
1590 self,
1591 source_butler: LimitedButler,
1592 source_refs: Iterable[DatasetRef],
1593 transfer: str = "auto",
1594 skip_missing: bool = True,
1595 register_dataset_types: bool = False,
1596 transfer_dimensions: bool = False,
1597 dry_run: bool = False,
1598 ) -> Collection[DatasetRef]:
1599 """Transfer datasets to this Butler from a run in another Butler.
1601 Parameters
1602 ----------
1603 source_butler : `LimitedButler`
1604 Butler from which the datasets are to be transferred. If data IDs
1605 in ``source_refs`` are not expanded then this has to be a full
1606 `Butler` whose registry will be used to expand data IDs.
1607 source_refs : iterable of `DatasetRef`
1608 Datasets defined in the source butler that should be transferred to
1609 this butler. In most circumstances, ``transfer_from`` is faster if
1610 the dataset refs are expanded.
1611 transfer : `str`, optional
1612 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1613 skip_missing : `bool`
1614 If `True`, datasets with no datastore artifact associated with
1615 them are not transferred. If `False` a registry entry will be
1616 created even if no datastore record is created (and so will
1617 look equivalent to the dataset being unstored).
1618 register_dataset_types : `bool`
1619 If `True` any missing dataset types are registered. Otherwise
1620 an exception is raised.
1621 transfer_dimensions : `bool`, optional
1622 If `True`, dimension record data associated with the new datasets
1623 will be transferred.
1624 dry_run : `bool`, optional
1625 If `True` the transfer will be processed without any modifications
1626 made to the target butler and as if the target butler did not
1627 have any of the datasets.
1629 Returns
1630 -------
1631 refs : `list` of `DatasetRef`
1632 The refs added to this Butler.
1634 Notes
1635 -----
1636 The datastore artifact has to exist for a transfer
1637 to be made but non-existence is not an error.
1639 Datasets that already exist in this run will be skipped.
1641 The datasets are imported as part of a transaction, although
1642 dataset types are registered before the transaction is started.
1643 This means that it is possible for a dataset type to be registered
1644 even though transfer has failed.
1645 """
1646 raise NotImplementedError()
1648 @abstractmethod
1649 def validateConfiguration(
1650 self,
1651 logFailures: bool = False,
1652 datasetTypeNames: Iterable[str] | None = None,
1653 ignore: Iterable[str] | None = None,
1654 ) -> None:
1655 """Validate butler configuration.
1657 Checks that each `DatasetType` can be stored in the `Datastore`.
1659 Parameters
1660 ----------
1661 logFailures : `bool`, optional
1662 If `True`, output a log message for every validation error
1663 detected.
1664 datasetTypeNames : iterable of `str`, optional
1665 The `DatasetType` names that should be checked. This allows
1666 only a subset to be selected.
1667 ignore : iterable of `str`, optional
1668 Names of DatasetTypes to skip over. This can be used to skip
1669 known problems. If a named `DatasetType` corresponds to a
1670 composite, all components of that `DatasetType` will also be
1671 ignored.
1673 Raises
1674 ------
1675 ButlerValidationError
1676 Raised if there is some inconsistency with how this Butler
1677 is configured.
1678 """
1679 raise NotImplementedError()
1681 @property
1682 @abstractmethod
1683 def collection_chains(self) -> ButlerCollections:
1684 """Object with methods for modifying collection chains
1685 (`~lsst.daf.butler.ButlerCollections`).
1687 Deprecated. Replaced with ``collections`` property.
1688 """
1689 raise NotImplementedError()
1691 @property
1692 @abstractmethod
1693 def collections(self) -> ButlerCollections:
1694 """Object with methods for modifying and querying collections
1695 (`~lsst.daf.butler.ButlerCollections`).
1697 Use of this object is preferred over `registry` wherever possible.
1698 """
1699 raise NotImplementedError()
1701 @property
1702 @abstractmethod
1703 def run(self) -> str | None:
1704 """Name of the run this butler writes outputs to by default (`str` or
1705 `None`).
1706 """
1707 raise NotImplementedError()
1709 @property
1710 @abstractmethod
1711 def registry(self) -> Registry:
1712 """The object that manages dataset metadata and relationships
1713 (`Registry`).
1715 Many operations that don't involve reading or writing butler datasets
1716 are accessible only via `Registry` methods. Eventually these methods
1717 will be replaced by equivalent `Butler` methods.
1718 """
1719 raise NotImplementedError()
1721 @abstractmethod
1722 def query(self) -> AbstractContextManager[Query]:
1723 """Context manager returning a `Query` object used for construction
1724 and execution of complex queries.
1725 """
1726 raise NotImplementedError()
1728 def query_data_ids(
1729 self,
1730 dimensions: DimensionGroup | Iterable[str] | str,
1731 *,
1732 data_id: DataId | None = None,
1733 where: str = "",
1734 bind: Mapping[str, Any] | None = None,
1735 with_dimension_records: bool = False,
1736 order_by: Iterable[str] | str | None = None,
1737 limit: int | None = -20_000,
1738 explain: bool = True,
1739 **kwargs: Any,
1740 ) -> list[DataCoordinate]:
1741 """Query for data IDs matching user-provided criteria.
1743 Parameters
1744 ----------
1745 dimensions : `DimensionGroup`, `str`, or \
1746 `~collections.abc.Iterable` [`str`]
1747 The dimensions of the data IDs to yield, as either `DimensionGroup`
1748 instances or `str`. Will be automatically expanded to a complete
1749 `DimensionGroup`.
1750 data_id : `dict` or `DataCoordinate`, optional
1751 A data ID whose key-value pairs are used as equality constraints
1752 in the query.
1753 where : `str`, optional
1754 A string expression similar to a SQL WHERE clause. May involve
1755 any column of a dimension table or (as a shortcut for the primary
1756 key column of a dimension table) dimension name. See
1757 :ref:`daf_butler_dimension_expressions` for more information.
1758 bind : `~collections.abc.Mapping`, optional
1759 Mapping containing literal values that should be injected into the
1760 ``where`` expression, keyed by the identifiers they replace.
1761 Values of collection type can be expanded in some cases; see
1762 :ref:`daf_butler_dimension_expressions_identifiers` for more
1763 information.
1764 with_dimension_records : `bool`, optional
1765 If `True` (default is `False`) then returned data IDs will have
1766 dimension records.
1767 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1768 Names of the columns/dimensions to use for ordering returned data
1769 IDs. Column name can be prefixed with minus (``-``) to use
1770 descending ordering.
1771 limit : `int` or `None`, optional
1772 Upper limit on the number of returned records. `None` can be used
1773 if no limit is wanted. A limit of ``0`` means that the query will
1774 be executed and validated but no results will be returned. In this
1775 case there will be no exception even if ``explain`` is `True`.
1776 If a negative value is given a warning will be issued if the number
1777 of results is capped by that limit.
1778 explain : `bool`, optional
1779 If `True` (default) then `EmptyQueryResultError` exception is
1780 raised when resulting list is empty. The exception contains
1781 non-empty list of strings explaining possible causes for empty
1782 result.
1783 **kwargs
1784 Additional keyword arguments are forwarded to
1785 `DataCoordinate.standardize` when processing the ``data_id``
1786 argument (and may be used to provide a constraining data ID even
1787 when the ``data_id`` argument is `None`).
1789 Returns
1790 -------
1791 dataIds : `list` [`DataCoordinate`]
1792 Data IDs matching the given query parameters. These are always
1793 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1794 returns `True`).
1796 Raises
1797 ------
1798 lsst.daf.butler.registry.DataIdError
1799 Raised when ``data_id`` or keyword arguments specify unknown
1800 dimensions or values, or when they contain inconsistent values.
1801 lsst.daf.butler.registry.UserExpressionError
1802 Raised when ``where`` expression is invalid.
1803 lsst.daf.butler.EmptyQueryResultError
1804 Raised when query generates empty result and ``explain`` is set to
1805 `True`.
1806 TypeError
1807 Raised when the arguments are incompatible.
1808 """
1809 if data_id is None:
1810 data_id = DataCoordinate.make_empty(self.dimensions)
1811 if order_by is None:
1812 order_by = []
1813 query_limit = limit
1814 warn_limit = False
1815 if limit is not None and limit < 0:
1816 query_limit = abs(limit) + 1
1817 warn_limit = True
1818 with self.query() as query:
1819 result = (
1820 query.data_ids(dimensions)
1821 .where(data_id, where, bind=bind, **kwargs)
1822 .order_by(*ensure_iterable(order_by))
1823 .limit(query_limit)
1824 )
1825 if with_dimension_records:
1826 result = result.with_dimension_records()
1827 data_ids = list(result)
1828 if warn_limit and len(data_ids) == query_limit:
1829 # We asked for one too many so must remove that from the list.
1830 data_ids.pop(-1)
1831 assert limit is not None # For mypy.
1832 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1833 if explain and (limit is None or limit != 0) and not data_ids:
1834 raise EmptyQueryResultError(list(result.explain_no_results()))
1835 return data_ids
1837 def query_datasets(
1838 self,
1839 dataset_type: str | DatasetType,
1840 collections: str | Iterable[str] | None = None,
1841 *,
1842 find_first: bool = True,
1843 data_id: DataId | None = None,
1844 where: str = "",
1845 bind: Mapping[str, Any] | None = None,
1846 with_dimension_records: bool = False,
1847 order_by: Iterable[str] | str | None = None,
1848 limit: int | None = -20_000,
1849 explain: bool = True,
1850 **kwargs: Any,
1851 ) -> list[DatasetRef]:
1852 """Query for dataset references matching user-provided criteria.
1854 Parameters
1855 ----------
1856 dataset_type : `str` or `DatasetType`
1857 Dataset type object or name to search for.
1858 collections : collection expression, optional
1859 A collection name or iterable of collection names to search. If not
1860 provided, the default collections are used. Can be a wildcard if
1861 ``find_first`` is `False` (if find first is requested the order
1862 of collections matters and wildcards make the order indeterminate).
1863 See :ref:`daf_butler_collection_expressions` for more information.
1864 find_first : `bool`, optional
1865 If `True` (default), for each result data ID, only yield one
1866 `DatasetRef` of each `DatasetType`, from the first collection in
1867 which a dataset of that dataset type appears (according to the
1868 order of ``collections`` passed in). If `True`, ``collections``
1869 must not contain wildcards.
1870 data_id : `dict` or `DataCoordinate`, optional
1871 A data ID whose key-value pairs are used as equality constraints in
1872 the query.
1873 where : `str`, optional
1874 A string expression similar to a SQL WHERE clause. May involve any
1875 column of a dimension table or (as a shortcut for the primary key
1876 column of a dimension table) dimension name. See
1877 :ref:`daf_butler_dimension_expressions` for more information.
1878 bind : `~collections.abc.Mapping`, optional
1879 Mapping containing literal values that should be injected into the
1880 ``where`` expression, keyed by the identifiers they replace. Values
1881 of collection type can be expanded in some cases; see
1882 :ref:`daf_butler_dimension_expressions_identifiers` for more
1883 information.
1884 with_dimension_records : `bool`, optional
1885 If `True` (default is `False`) then returned data IDs will have
1886 dimension records.
1887 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1888 Names of the columns/dimensions to use for ordering returned data
1889 IDs. Column name can be prefixed with minus (``-``) to use
1890 descending ordering.
1891 limit : `int` or `None`, optional
1892 Upper limit on the number of returned records. `None` can be used
1893 if no limit is wanted. A limit of ``0`` means that the query will
1894 be executed and validated but no results will be returned. In this
1895 case there will be no exception even if ``explain`` is `True`.
1896 If a negative value is given a warning will be issued if the number
1897 of results is capped by that limit.
1898 explain : `bool`, optional
1899 If `True` (default) then `EmptyQueryResultError` exception is
1900 raised when resulting list is empty. The exception contains
1901 non-empty list of strings explaining possible causes for empty
1902 result.
1903 **kwargs
1904 Additional keyword arguments are forwarded to
1905 `DataCoordinate.standardize` when processing the ``data_id``
1906 argument (and may be used to provide a constraining data ID even
1907 when the ``data_id`` argument is `None`).
1909 Returns
1910 -------
1911 refs : `.queries.DatasetRefQueryResults`
1912 Dataset references matching the given query criteria. Nested data
1913 IDs are guaranteed to include values for all implied dimensions
1914 (i.e. `DataCoordinate.hasFull` will return `True`).
1916 Raises
1917 ------
1918 lsst.daf.butler.registry.DatasetTypeExpressionError
1919 Raised when ``dataset_type`` expression is invalid.
1920 lsst.daf.butler.registry.DataIdError
1921 Raised when ``data_id`` or keyword arguments specify unknown
1922 dimensions or values, or when they contain inconsistent values.
1923 lsst.daf.butler.registry.UserExpressionError
1924 Raised when ``where`` expression is invalid.
1925 lsst.daf.butler.EmptyQueryResultError
1926 Raised when query generates empty result and ``explain`` is set to
1927 `True`.
1928 TypeError
1929 Raised when the arguments are incompatible, such as when a
1930 collection wildcard is passed when ``find_first`` is `True`, or
1931 when ``collections`` is `None` and default butler collections are
1932 not defined.
1933 """
1934 if data_id is None:
1935 data_id = DataCoordinate.make_empty(self.dimensions)
1936 if order_by is None:
1937 order_by = []
1938 if collections and has_globs(collections):
1939 # Wild cards need to be expanded but can only be allowed if
1940 # find_first=False because expanding wildcards does not return
1941 # a guaranteed ordering. Querying collection registry to expand
1942 # collections when we do not have wildcards is expensive so only
1943 # do it if we need it.
1944 if find_first:
1945 raise InvalidQueryError(
1946 f"Can not use wildcards in collections when find_first=True (given {collections})"
1947 )
1948 collections = self.collections.query(collections)
1949 query_limit = limit
1950 warn_limit = False
1951 if limit is not None and limit < 0:
1952 query_limit = abs(limit) + 1
1953 warn_limit = True
1954 with self.query() as query:
1955 result = (
1956 query.datasets(dataset_type, collections=collections, find_first=find_first)
1957 .where(data_id, where, bind=bind, **kwargs)
1958 .order_by(*ensure_iterable(order_by))
1959 .limit(query_limit)
1960 )
1961 if with_dimension_records:
1962 result = result.with_dimension_records()
1963 refs = list(result)
1964 if warn_limit and len(refs) == query_limit:
1965 # We asked for one too many so must remove that from the list.
1966 refs.pop(-1)
1967 assert limit is not None # For mypy.
1968 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1969 if explain and (limit is None or limit != 0) and not refs:
1970 raise EmptyQueryResultError(list(result.explain_no_results()))
1971 return refs
1973 def query_dimension_records(
1974 self,
1975 element: str,
1976 *,
1977 data_id: DataId | None = None,
1978 where: str = "",
1979 bind: Mapping[str, Any] | None = None,
1980 order_by: Iterable[str] | str | None = None,
1981 limit: int | None = -20_000,
1982 explain: bool = True,
1983 **kwargs: Any,
1984 ) -> list[DimensionRecord]:
1985 """Query for dimension information matching user-provided criteria.
1987 Parameters
1988 ----------
1989 element : `str`
1990 The name of a dimension element to obtain records for.
1991 data_id : `dict` or `DataCoordinate`, optional
1992 A data ID whose key-value pairs are used as equality constraints
1993 in the query.
1994 where : `str`, optional
1995 A string expression similar to a SQL WHERE clause. See
1996 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1997 information.
1998 bind : `~collections.abc.Mapping`, optional
1999 Mapping containing literal values that should be injected into the
2000 ``where`` expression, keyed by the identifiers they replace.
2001 Values of collection type can be expanded in some cases; see
2002 :ref:`daf_butler_dimension_expressions_identifiers` for more
2003 information.
2004 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2005 Names of the columns/dimensions to use for ordering returned data
2006 IDs. Column name can be prefixed with minus (``-``) to use
2007 descending ordering.
2008 limit : `int` or `None`, optional
2009 Upper limit on the number of returned records. `None` can be used
2010 if no limit is wanted. A limit of ``0`` means that the query will
2011 be executed and validated but no results will be returned. In this
2012 case there will be no exception even if ``explain`` is `True`.
2013 If a negative value is given a warning will be issued if the number
2014 of results is capped by that limit.
2015 explain : `bool`, optional
2016 If `True` (default) then `EmptyQueryResultError` exception is
2017 raised when resulting list is empty. The exception contains
2018 non-empty list of strings explaining possible causes for empty
2019 result.
2020 **kwargs
2021 Additional keyword arguments are forwarded to
2022 `DataCoordinate.standardize` when processing the ``data_id``
2023 argument (and may be used to provide a constraining data ID even
2024 when the ``data_id`` argument is `None`).
2026 Returns
2027 -------
2028 records : `list`[`DimensionRecord`]
2029 Dimension records matching the given query parameters.
2031 Raises
2032 ------
2033 lsst.daf.butler.registry.DataIdError
2034 Raised when ``data_id`` or keyword arguments specify unknown
2035 dimensions or values, or when they contain inconsistent values.
2036 lsst.daf.butler.registry.UserExpressionError
2037 Raised when ``where`` expression is invalid.
2038 lsst.daf.butler.EmptyQueryResultError
2039 Raised when query generates empty result and ``explain`` is set to
2040 `True`.
2041 TypeError
2042 Raised when the arguments are incompatible, such as when a
2043 collection wildcard is passed when ``find_first`` is `True`, or
2044 when ``collections`` is `None` and default butler collections are
2045 not defined.
2046 """
2047 if data_id is None:
2048 data_id = DataCoordinate.make_empty(self.dimensions)
2049 if order_by is None:
2050 order_by = []
2051 query_limit = limit
2052 warn_limit = False
2053 if limit is not None and limit < 0:
2054 query_limit = abs(limit) + 1
2055 warn_limit = True
2056 with self.query() as query:
2057 result = (
2058 query.dimension_records(element)
2059 .where(data_id, where, bind=bind, **kwargs)
2060 .order_by(*ensure_iterable(order_by))
2061 .limit(query_limit)
2062 )
2063 dimension_records = list(result)
2064 if warn_limit and len(dimension_records) == query_limit:
2065 # We asked for one too many so must remove that from the list.
2066 dimension_records.pop(-1)
2067 assert limit is not None # For mypy.
2068 _LOG.warning(
2069 "More dimension records are available than the requested limit of %d.", abs(limit)
2070 )
2071 if explain and (limit is None or limit != 0) and not dimension_records:
2072 raise EmptyQueryResultError(list(result.explain_no_results()))
2073 return dimension_records
2075 def query_all_datasets(
2076 self,
2077 collections: str | Iterable[str] | None = None,
2078 *,
2079 name: str | Iterable[str] = "*",
2080 find_first: bool = True,
2081 data_id: DataId | None = None,
2082 where: str = "",
2083 bind: Mapping[str, Any] | None = None,
2084 limit: int | None = -20_000,
2085 **kwargs: Any,
2086 ) -> list[DatasetRef]:
2087 """Query for datasets of potentially multiple types.
2089 Parameters
2090 ----------
2091 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2092 The collection or collections to search, in order. If not provided
2093 or `None`, the default collection search path for this butler is
2094 used.
2095 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2096 Names or name patterns (glob-style) that returned dataset type
2097 names must match. If an iterable, items are OR'd together. The
2098 default is to include all dataset types in the given collections.
2099 find_first : `bool`, optional
2100 If `True` (default), for each result data ID, only yield one
2101 `DatasetRef` of each `DatasetType`, from the first collection in
2102 which a dataset of that dataset type appears (according to the
2103 order of ``collections`` passed in).
2104 data_id : `dict` or `DataCoordinate`, optional
2105 A data ID whose key-value pairs are used as equality constraints in
2106 the query.
2107 where : `str`, optional
2108 A string expression similar to a SQL WHERE clause. May involve any
2109 column of a dimension table or (as a shortcut for the primary key
2110 column of a dimension table) dimension name. See
2111 :ref:`daf_butler_dimension_expressions` for more information.
2112 bind : `~collections.abc.Mapping`, optional
2113 Mapping containing literal values that should be injected into the
2114 ``where`` expression, keyed by the identifiers they replace. Values
2115 of collection type can be expanded in some cases; see
2116 :ref:`daf_butler_dimension_expressions_identifiers` for more
2117 information.
2118 limit : `int` or `None`, optional
2119 Upper limit on the number of returned records. `None` can be used
2120 if no limit is wanted. A limit of ``0`` means that the query will
2121 be executed and validated but no results will be returned.
2122 If a negative value is given a warning will be issued if the number
2123 of results is capped by that limit. If no limit is provided, by
2124 default a maximum of 20,000 records will be returned.
2125 **kwargs
2126 Additional keyword arguments are forwarded to
2127 `DataCoordinate.standardize` when processing the ``data_id``
2128 argument (and may be used to provide a constraining data ID even
2129 when the ``data_id`` argument is `None`).
2131 Raises
2132 ------
2133 MissingDatasetTypeError
2134 When no dataset types match ``name``, or an explicit (non-glob)
2135 dataset type in ``name`` does not exist.
2136 InvalidQueryError
2137 If the parameters to the query are inconsistent or malformed.
2138 MissingCollectionError
2139 If a given collection is not found.
2141 Returns
2142 -------
2143 refs : `list` [ `DatasetRef` ]
2144 Dataset references matching the given query criteria. Nested data
2145 IDs are guaranteed to include values for all implied dimensions
2146 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2147 include dimension records (`DataCoordinate.hasRecords` will be
2148 `False`).
2149 """
2150 if collections is None:
2151 collections = list(self.collections.defaults)
2152 else:
2153 collections = list(ensure_iterable(collections))
2155 if bind is None:
2156 bind = {}
2157 if data_id is None:
2158 data_id = {}
2160 warn_limit = False
2161 if limit is not None and limit < 0:
2162 # Add one to the limit so we can detect if we have exceeded it.
2163 limit = abs(limit) + 1
2164 warn_limit = True
2166 args = QueryAllDatasetsParameters(
2167 collections=collections,
2168 name=list(ensure_iterable(name)),
2169 find_first=find_first,
2170 data_id=data_id,
2171 where=where,
2172 limit=limit,
2173 bind=bind,
2174 kwargs=kwargs,
2175 with_dimension_records=False,
2176 )
2177 with self._query_all_datasets_by_page(args) as pages:
2178 result = []
2179 for page in pages:
2180 result.extend(page)
2182 if warn_limit and limit is not None and len(result) >= limit:
2183 # Remove the extra dataset we added for the limit check.
2184 result.pop()
2185 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2187 return result
2189 @abstractmethod
2190 def _query_all_datasets_by_page(
2191 self, args: QueryAllDatasetsParameters
2192 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2193 raise NotImplementedError()
2195 def clone(
2196 self,
2197 *,
2198 collections: CollectionArgType | None | EllipsisType = ...,
2199 run: str | None | EllipsisType = ...,
2200 inferDefaults: bool | EllipsisType = ...,
2201 dataId: dict[str, str] | EllipsisType = ...,
2202 metrics: ButlerMetrics | None = None,
2203 ) -> Butler:
2204 """Return a new Butler instance connected to the same repository
2205 as this one, optionally overriding ``collections``, ``run``,
2206 ``inferDefaults``, and default data ID.
2208 Parameters
2209 ----------
2210 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2211 optional
2212 Same as constructor. If omitted, uses value from original object.
2213 run : `str` or `None`, optional
2214 Same as constructor. If `None`, no default run is used. If
2215 omitted, copies value from original object.
2216 inferDefaults : `bool`, optional
2217 Same as constructor. If omitted, copies value from original
2218 object.
2219 dataId : `str`
2220 Same as ``kwargs`` passed to the constructor. If omitted, copies
2221 values from original object.
2222 metrics : `ButlerMetrics` or `None`, optional
2223 Metrics object to record butler statistics.
2224 """
2225 raise NotImplementedError()
2227 @abstractmethod
2228 def close(self) -> None:
2229 raise NotImplementedError()