Coverage for python / lsst / daf / butler / _butler.py: 30%
310 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-15 06:22 -0300
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35from abc import abstractmethod
36from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
37from contextlib import AbstractContextManager
38from types import EllipsisType
39from typing import TYPE_CHECKING, Any, TextIO
41from lsst.resources import ResourcePath, ResourcePathExpression
42from lsst.utils import doImportType
43from lsst.utils.iteration import ensure_iterable
44from lsst.utils.logging import getLogger
46from ._butler_collections import ButlerCollections
47from ._butler_config import ButlerConfig, ButlerType
48from ._butler_instance_options import ButlerInstanceOptions
49from ._butler_metrics import ButlerMetrics
50from ._butler_repo_index import ButlerRepoIndex
51from ._config import Config, ConfigSubset
52from ._exceptions import EmptyQueryResultError, InvalidQueryError
53from ._limited_butler import LimitedButler
54from ._query_all_datasets import QueryAllDatasetsParameters
55from .datastore import Datastore
56from .dimensions import DataCoordinate, DimensionConfig
57from .registry import RegistryConfig, _RegistryFactory
58from .repo_relocation import BUTLER_ROOT_TAG
59from .utils import has_globs
61if TYPE_CHECKING:
62 from ._dataset_existence import DatasetExistence
63 from ._dataset_provenance import DatasetProvenance
64 from ._dataset_ref import DatasetId, DatasetRef
65 from ._dataset_type import DatasetType
66 from ._deferredDatasetHandle import DeferredDatasetHandle
67 from ._file_dataset import FileDataset
68 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
69 from ._storage_class import StorageClass
70 from ._timespan import Timespan
71 from .datastore import DatasetRefURIs
72 from .dimensions import DataId, DimensionGroup, DimensionRecord
73 from .queries import Query
74 from .registry import CollectionArgType, Registry
75 from .transfers import RepoExportContext
77_LOG = getLogger(__name__)
80@dataclasses.dataclass
81class ParsedButlerDatasetURI:
82 label: str
83 dataset_id: uuid.UUID
84 uri: str
87@dataclasses.dataclass
88class SpecificButlerDataset:
89 butler: Butler
90 dataset: DatasetRef | None
93class _DeprecatedDefault:
94 """Default value for a deprecated parameter."""
97class Butler(LimitedButler): # numpydoc ignore=PR02
98 """Interface for data butler and factory for Butler instances.
100 Parameters
101 ----------
102 config : `ButlerConfig`, `Config` or `str`, optional
103 Configuration. Anything acceptable to the `ButlerConfig` constructor.
104 If a directory path is given the configuration will be read from a
105 ``butler.yaml`` file in that location. If `None` is given default
106 values will be used. If ``config`` contains "cls" key then its value is
107 used as a name of butler class and it must be a sub-class of this
108 class, otherwise `DirectButler` is instantiated.
109 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
110 An expression specifying the collections to be searched (in order) when
111 reading datasets.
112 This may be a `str` collection name or an iterable thereof.
113 See :ref:`daf_butler_collection_expressions` for more information.
114 These collections are not registered automatically and must be
115 manually registered before they are used by any method, but they may be
116 manually registered after the `Butler` is initialized.
117 run : `str`, optional
118 Name of the `~CollectionType.RUN` collection new datasets should be
119 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
120 ``collections`` will be set to ``[run]``. If not `None`, this
121 collection will automatically be registered. If this is not set (and
122 ``writeable`` is not set either), a read-only butler will be created.
123 searchPaths : `list` of `str`, optional
124 Directory paths to search when calculating the full Butler
125 configuration. Not used if the supplied config is already a
126 `ButlerConfig`.
127 writeable : `bool`, optional
128 Explicitly sets whether the butler supports write operations. If not
129 provided, a read-write butler is created if any of ``run``, ``tags``,
130 or ``chains`` is non-empty.
131 inferDefaults : `bool`, optional
132 If `True` (default) infer default data ID values from the values
133 present in the datasets in ``collections``: if all collections have the
134 same value (or no value) for a governor dimension, that value will be
135 the default for that dimension. Nonexistent collections are ignored.
136 If a default value is provided explicitly for a governor dimension via
137 ``**kwargs``, no default will be inferred for that dimension.
138 without_datastore : `bool`, optional
139 If `True` do not attach a datastore to this butler. Any attempts
140 to use a datastore will fail.
141 **kwargs : `Any`
142 Additional keyword arguments passed to a constructor of actual butler
143 class.
145 Notes
146 -----
147 The preferred way to instantiate Butler is via the `from_config` method.
148 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
149 but ``mypy`` will complain about the former.
150 """
152 def __new__(
153 cls,
154 config: Config | ResourcePathExpression | None = None,
155 *,
156 collections: Any = None,
157 run: str | None = None,
158 searchPaths: Sequence[ResourcePathExpression] | None = None,
159 writeable: bool | None = None,
160 inferDefaults: bool = True,
161 without_datastore: bool = False,
162 metrics: ButlerMetrics | None = None,
163 **kwargs: Any,
164 ) -> Butler:
165 if cls is Butler:
166 return Butler.from_config(
167 config=config,
168 collections=collections,
169 run=run,
170 searchPaths=searchPaths,
171 writeable=writeable,
172 inferDefaults=inferDefaults,
173 without_datastore=without_datastore,
174 metrics=metrics,
175 **kwargs,
176 )
178 # Note: we do not pass any parameters to __new__, Python will pass them
179 # to __init__ after __new__ returns sub-class instance.
180 return super().__new__(cls)
182 @classmethod
183 def from_config(
184 cls,
185 config: Config | ResourcePathExpression | None = None,
186 *,
187 collections: Any = None,
188 run: str | None = None,
189 searchPaths: Sequence[ResourcePathExpression] | None = None,
190 writeable: bool | None = None,
191 inferDefaults: bool = True,
192 without_datastore: bool = False,
193 metrics: ButlerMetrics | None = None,
194 **kwargs: Any,
195 ) -> Butler:
196 """Create butler instance from configuration.
198 Parameters
199 ----------
200 config : `ButlerConfig`, `Config` or `str`, optional
201 Configuration. Anything acceptable to the `ButlerConfig`
202 constructor. If a directory path is given the configuration will be
203 read from a ``butler.yaml`` file in that location. If `None` is
204 given default values will be used. If ``config`` contains "cls" key
205 then its value is used as a name of butler class and it must be a
206 sub-class of this class, otherwise `DirectButler` is instantiated.
207 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
208 An expression specifying the collections to be searched (in order)
209 when reading datasets.
210 This may be a `str` collection name or an iterable thereof.
211 See :ref:`daf_butler_collection_expressions` for more information.
212 These collections are not registered automatically and must be
213 manually registered before they are used by any method, but they
214 may be manually registered after the `Butler` is initialized.
215 run : `str`, optional
216 Name of the `~CollectionType.RUN` collection new datasets should be
217 inserted into. If ``collections`` is `None` and ``run`` is not
218 `None`, ``collections`` will be set to ``[run]``. If not `None`,
219 this collection will automatically be registered. If this is not
220 set (and ``writeable`` is not set either), a read-only butler will
221 be created.
222 searchPaths : `list` of `str`, optional
223 Directory paths to search when calculating the full Butler
224 configuration. Not used if the supplied config is already a
225 `ButlerConfig`.
226 writeable : `bool`, optional
227 Explicitly sets whether the butler supports write operations. If
228 not provided, a read-write butler is created if any of ``run``,
229 ``tags``, or ``chains`` is non-empty.
230 inferDefaults : `bool`, optional
231 If `True` (default) infer default data ID values from the values
232 present in the datasets in ``collections``: if all collections have
233 the same value (or no value) for a governor dimension, that value
234 will be the default for that dimension. Nonexistent collections
235 are ignored. If a default value is provided explicitly for a
236 governor dimension via ``**kwargs``, no default will be inferred
237 for that dimension.
238 without_datastore : `bool`, optional
239 If `True` do not attach a datastore to this butler. Any attempts
240 to use a datastore will fail.
241 metrics : `ButlerMetrics` or `None`, optional
242 Metrics object to record butler usage statistics.
243 **kwargs : `Any`
244 Default data ID key-value pairs. These may only identify
245 "governor" dimensions like ``instrument`` and ``skymap``.
247 Returns
248 -------
249 butler : `Butler`
250 A `Butler` constructed from the given configuration.
252 Notes
253 -----
254 Calling this factory method is identical to calling
255 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
256 complains about ``Butler()`` call.
258 Examples
259 --------
260 While there are many ways to control exactly how a `Butler` interacts
261 with the collections in its `Registry`, the most common cases are still
262 simple.
264 For a read-only `Butler` that searches one collection, do::
266 butler = Butler.from_config(
267 "/path/to/repo", collections=["u/alice/DM-50000"]
268 )
270 For a read-write `Butler` that writes to and reads from a
271 `~CollectionType.RUN` collection::
273 butler = Butler.from_config(
274 "/path/to/repo", run="u/alice/DM-50000/a"
275 )
277 The `Butler` passed to a ``PipelineTask`` is often much more complex,
278 because we want to write to one `~CollectionType.RUN` collection but
279 read from several others (as well)::
281 butler = Butler.from_config(
282 "/path/to/repo",
283 run="u/alice/DM-50000/a",
284 collections=[
285 "u/alice/DM-50000/a",
286 "u/bob/DM-49998",
287 "HSC/defaults",
288 ],
289 )
291 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
292 Datasets will be read first from that run (since it appears first in
293 the chain), and then from ``u/bob/DM-49998`` and finally
294 ``HSC/defaults``.
296 Finally, one can always create a `Butler` with no collections::
298 butler = Butler.from_config("/path/to/repo", writeable=True)
300 This can be extremely useful when you just want to use
301 ``butler.registry``, e.g. for inserting dimension data or managing
302 collections, or when the collections you want to use with the butler
303 are not consistent. Passing ``writeable`` explicitly here is only
304 necessary if you want to be able to make changes to the repo - usually
305 the value for ``writeable`` can be guessed from the collection
306 arguments provided, but it defaults to `False` when there are not
307 collection arguments.
308 """
309 # DirectButler used to have a way to specify a "copy constructor" by
310 # passing the "butler" parameter to its constructor. This has
311 # been moved out of the constructor into Butler.clone().
312 butler = kwargs.pop("butler", None)
313 metrics = metrics if metrics is not None else ButlerMetrics()
314 if butler is not None:
315 if not isinstance(butler, Butler):
316 raise TypeError("'butler' parameter must be a Butler instance")
317 if config is not None or searchPaths is not None or writeable is not None:
318 raise TypeError(
319 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
320 )
321 return butler.clone(
322 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
323 )
325 options = ButlerInstanceOptions(
326 collections=collections,
327 run=run,
328 writeable=writeable,
329 inferDefaults=inferDefaults,
330 metrics=metrics,
331 kwargs=kwargs,
332 )
334 # Load the Butler configuration. This may involve searching the
335 # environment to locate a configuration file.
336 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
337 butler_type = butler_config.get_butler_type()
339 # Make DirectButler if class is not specified.
340 match butler_type:
341 case ButlerType.DIRECT:
342 from .direct_butler import DirectButler
344 return DirectButler.create_from_config(
345 butler_config,
346 options=options,
347 without_datastore=without_datastore,
348 )
349 case ButlerType.REMOTE:
350 from .remote_butler._factory import RemoteButlerFactory
352 # Assume this is being created by a client who would like
353 # default caching of remote datasets.
354 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
355 return factory.create_butler_with_credentials_from_environment(
356 butler_options=options, enable_datastore_cache=True
357 )
358 case _:
359 raise TypeError(f"Unknown Butler type '{butler_type}'")
361 @staticmethod
362 def makeRepo(
363 root: ResourcePathExpression,
364 config: Config | str | None = None,
365 dimensionConfig: Config | str | None = None,
366 standalone: bool = False,
367 searchPaths: list[str] | None = None,
368 forceConfigRoot: bool = True,
369 outfile: ResourcePathExpression | None = None,
370 overwrite: bool = False,
371 ) -> Config:
372 """Create an empty data repository by adding a butler.yaml config
373 to a repository root directory.
375 Parameters
376 ----------
377 root : `lsst.resources.ResourcePathExpression`
378 Path or URI to the root location of the new repository. Will be
379 created if it does not exist.
380 config : `Config` or `str`, optional
381 Configuration to write to the repository, after setting any
382 root-dependent Registry or Datastore config options. Can not
383 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
384 configuration will be used. Root-dependent config options
385 specified in this config are overwritten if ``forceConfigRoot``
386 is `True`.
387 dimensionConfig : `Config` or `str`, optional
388 Configuration for dimensions, will be used to initialize registry
389 database.
390 standalone : `bool`
391 If True, write all expanded defaults, not just customized or
392 repository-specific settings.
393 This (mostly) decouples the repository from the default
394 configuration, insulating it from changes to the defaults (which
395 may be good or bad, depending on the nature of the changes).
396 Future *additions* to the defaults will still be picked up when
397 initializing `Butlers` to repos created with ``standalone=True``.
398 searchPaths : `list` of `str`, optional
399 Directory paths to search when calculating the full butler
400 configuration.
401 forceConfigRoot : `bool`, optional
402 If `False`, any values present in the supplied ``config`` that
403 would normally be reset are not overridden and will appear
404 directly in the output config. This allows non-standard overrides
405 of the root directory for a datastore or registry to be given.
406 If this parameter is `True` the values for ``root`` will be
407 forced into the resulting config if appropriate.
408 outfile : `lss.resources.ResourcePathExpression`, optional
409 If not-`None`, the output configuration will be written to this
410 location rather than into the repository itself. Can be a URI
411 string. Can refer to a directory that will be used to write
412 ``butler.yaml``.
413 overwrite : `bool`, optional
414 Create a new configuration file even if one already exists
415 in the specified output location. Default is to raise
416 an exception.
418 Returns
419 -------
420 config : `Config`
421 The updated `Config` instance written to the repo.
423 Raises
424 ------
425 ValueError
426 Raised if a ButlerConfig or ConfigSubset is passed instead of a
427 regular Config (as these subclasses would make it impossible to
428 support ``standalone=False``).
429 FileExistsError
430 Raised if the output config file already exists.
431 os.error
432 Raised if the directory does not exist, exists but is not a
433 directory, or cannot be created.
435 Notes
436 -----
437 Note that when ``standalone=False`` (the default), the configuration
438 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
439 construct the repository should also be used to construct any Butlers
440 to avoid configuration inconsistencies.
441 """
442 if isinstance(config, ButlerConfig | ConfigSubset):
443 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
445 # Ensure that the root of the repository exists or can be made
446 root_uri = ResourcePath(root, forceDirectory=True)
447 root_uri.mkdir()
449 config = Config(config)
451 # If we are creating a new repo from scratch with relative roots,
452 # do not propagate an explicit root from the config file
453 if "root" in config:
454 del config["root"]
456 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
457 imported_class = doImportType(full["datastore", "cls"])
458 if not issubclass(imported_class, Datastore):
459 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
460 datastoreClass: type[Datastore] = imported_class
461 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
463 # if key exists in given config, parse it, otherwise parse the defaults
464 # in the expanded config
465 if config.get(("registry", "db")):
466 registryConfig = RegistryConfig(config)
467 else:
468 registryConfig = RegistryConfig(full)
469 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
470 if defaultDatabaseUri is not None:
471 Config.updateParameters(
472 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
473 )
474 else:
475 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
477 if standalone:
478 config.merge(full)
479 else:
480 # Always expand the registry.managers section into the per-repo
481 # config, because after the database schema is created, it's not
482 # allowed to change anymore. Note that in the standalone=True
483 # branch, _everything_ in the config is expanded, so there's no
484 # need to special case this.
485 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
486 configURI: ResourcePathExpression
487 if outfile is not None:
488 # When writing to a separate location we must include
489 # the root of the butler repo in the config else it won't know
490 # where to look.
491 config["root"] = root_uri.geturl()
492 configURI = outfile
493 else:
494 configURI = root_uri
495 # Strip obscore configuration, if it is present, before writing config
496 # to a file, obscore config will be stored in registry.
497 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
498 config_to_write = config.copy()
499 del config_to_write[obscore_config_key]
500 config_to_write.dumpToUri(configURI, overwrite=overwrite)
501 # configFile attribute is updated, need to copy it to original.
502 config.configFile = config_to_write.configFile
503 else:
504 config.dumpToUri(configURI, overwrite=overwrite)
506 # Create Registry and populate tables
507 registryConfig = RegistryConfig(config.get("registry"))
508 dimensionConfig = DimensionConfig(dimensionConfig)
509 registry = _RegistryFactory(registryConfig).create_from_config(
510 dimensionConfig=dimensionConfig, butlerRoot=root_uri
511 )
512 registry.close()
514 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
516 return config
518 @classmethod
519 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
520 """Look up the label in a butler repository index.
522 Parameters
523 ----------
524 label : `str`
525 Label of the Butler repository to look up.
526 return_label : `bool`, optional
527 If ``label`` cannot be found in the repository index (either
528 because index is not defined or ``label`` is not in the index) and
529 ``return_label`` is `True` then return ``ResourcePath(label)``.
530 If ``return_label`` is `False` (default) then an exception will be
531 raised instead.
533 Returns
534 -------
535 uri : `lsst.resources.ResourcePath`
536 URI to the Butler repository associated with the given label or
537 default value if it is provided.
539 Raises
540 ------
541 KeyError
542 Raised if the label is not found in the index, or if an index
543 is not defined, and ``return_label`` is `False`.
545 Notes
546 -----
547 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
548 information is discovered.
549 """
550 return ButlerRepoIndex.get_repo_uri(label, return_label)
552 @classmethod
553 def get_known_repos(cls) -> set[str]:
554 """Retrieve the list of known repository labels.
556 Returns
557 -------
558 repos : `set` of `str`
559 All the known labels. Can be empty if no index can be found.
561 Notes
562 -----
563 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
564 information is discovered.
565 """
566 return ButlerRepoIndex.get_known_repos()
568 @classmethod
569 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
570 """Extract the butler label and dataset ID from a dataset URI.
572 Parameters
573 ----------
574 uri : `str`
575 The dataset URI to parse.
577 Returns
578 -------
579 parsed : `ParsedButlerDatasetURI`
580 The label associated with the butler repository from which this
581 dataset originates and the ID of the dataset.
583 Notes
584 -----
585 Supports dataset URIs of the forms
586 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
587 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
588 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
589 can include anything supported by the `Butler` constructor, including
590 paths to repositories and alias labels.
592 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
594 will return a label of ``/repo/main``.
596 This method does not attempt to check that the dataset exists in the
597 labeled butler.
599 Since the IVOID can be issued by any publisher to represent a Butler
600 dataset there is no validation of the path or netloc component of the
601 URI. The only requirement is that there are ``id`` and ``repo`` keys
602 in the ``ivo`` URI query component.
603 """
604 parsed = urllib.parse.urlparse(uri)
605 parsed_scheme = parsed.scheme.lower()
606 if parsed_scheme == "ivo":
607 # Do not validate the netloc or the path values.
608 qs = urllib.parse.parse_qs(parsed.query)
609 if "repo" not in qs or "id" not in qs:
610 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
611 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
612 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
613 label = qs["repo"][0]
614 id_ = qs["id"][0]
615 elif parsed_scheme == "butler":
616 label = parsed.netloc # Butler label is case sensitive.
617 # Need to strip the leading /.
618 id_ = parsed.path[1:]
619 else:
620 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
621 # Strip trailing/leading whitespace from label.
622 label = label.strip()
623 if not label:
624 raise ValueError(f"No butler repository label found in uri {uri!r}")
625 try:
626 dataset_id = uuid.UUID(hex=id_)
627 except Exception as e:
628 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
629 raise
631 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
633 @classmethod
634 def get_dataset_from_uri(
635 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
636 ) -> SpecificButlerDataset:
637 """Get the dataset associated with the given dataset URI.
639 Parameters
640 ----------
641 uri : `str`
642 The URI associated with a dataset.
643 factory : `LabeledButlerFactoryProtocol` or `None`, optional
644 Bound factory function that will be given the butler label
645 and receive a `Butler`. If this is not provided the label
646 will be tried directly.
648 Returns
649 -------
650 result : `SpecificButlerDataset`
651 The butler associated with this URI and the dataset itself.
652 The dataset can be `None` if the UUID is valid but the dataset
653 is not known to this butler.
654 """
655 parsed = cls.parse_dataset_uri(uri)
656 butler: Butler | None = None
657 if factory is not None:
658 # If the label is not recognized, it might be a path.
659 try:
660 butler = factory(parsed.label)
661 except KeyError:
662 pass
663 if butler is None:
664 butler = cls.from_config(parsed.label)
665 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
667 @abstractmethod
668 def _caching_context(self) -> AbstractContextManager[None]:
669 """Context manager that enables caching."""
670 raise NotImplementedError()
672 @abstractmethod
673 def transaction(self) -> AbstractContextManager[None]:
674 """Context manager supporting `Butler` transactions.
676 Transactions can be nested.
677 """
678 raise NotImplementedError()
680 @abstractmethod
681 def put(
682 self,
683 obj: Any,
684 datasetRefOrType: DatasetRef | DatasetType | str,
685 /,
686 dataId: DataId | None = None,
687 *,
688 run: str | None = None,
689 provenance: DatasetProvenance | None = None,
690 **kwargs: Any,
691 ) -> DatasetRef:
692 """Store and register a dataset.
694 Parameters
695 ----------
696 obj : `object`
697 The dataset.
698 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
699 When `DatasetRef` is provided, ``dataId`` should be `None`.
700 Otherwise the `DatasetType` or name thereof. If a fully resolved
701 `DatasetRef` is given the run and ID are used directly.
702 dataId : `dict` or `DataCoordinate`
703 A `dict` of `Dimension` link name, value pairs that label the
704 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
705 should be provided as the second argument.
706 run : `str`, optional
707 The name of the run the dataset should be added to, overriding
708 ``self.run``. Not used if a resolved `DatasetRef` is provided.
709 provenance : `DatasetProvenance` or `None`, optional
710 Any provenance that should be attached to the serialized dataset.
711 Not supported by all serialization mechanisms.
712 **kwargs
713 Additional keyword arguments used to augment or construct a
714 `DataCoordinate`. See `DataCoordinate.standardize`
715 parameters. Not used if a resolve `DatasetRef` is provided.
717 Returns
718 -------
719 ref : `DatasetRef`
720 A reference to the stored dataset, updated with the correct id if
721 given.
723 Raises
724 ------
725 TypeError
726 Raised if the butler is read-only or if no run has been provided.
727 """
728 raise NotImplementedError()
730 @abstractmethod
731 def getDeferred(
732 self,
733 datasetRefOrType: DatasetRef | DatasetType | str,
734 /,
735 dataId: DataId | None = None,
736 *,
737 parameters: dict | None = None,
738 collections: Any = None,
739 storageClass: str | StorageClass | None = None,
740 timespan: Timespan | None = None,
741 **kwargs: Any,
742 ) -> DeferredDatasetHandle:
743 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
744 after an immediate registry lookup.
746 Parameters
747 ----------
748 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
749 When `DatasetRef` the `dataId` should be `None`.
750 Otherwise the `DatasetType` or name thereof.
751 dataId : `dict` or `DataCoordinate`, optional
752 A `dict` of `Dimension` link name, value pairs that label the
753 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
754 should be provided as the first argument.
755 parameters : `dict`
756 Additional StorageClass-defined options to control reading,
757 typically used to efficiently read only a subset of the dataset.
758 collections : Any, optional
759 Collections to be searched, overriding ``self.collections``.
760 Can be any of the types supported by the ``collections`` argument
761 to butler construction.
762 storageClass : `StorageClass` or `str`, optional
763 The storage class to be used to override the Python type
764 returned by this method. By default the returned type matches
765 the dataset type definition for this dataset. Specifying a
766 read `StorageClass` can force a different type to be returned.
767 This type must be compatible with the original type.
768 timespan : `Timespan` or `None`, optional
769 A timespan that the validity range of the dataset must overlap.
770 If not provided and this is a calibration dataset type, an attempt
771 will be made to find the timespan from any temporal coordinate
772 in the data ID.
773 **kwargs
774 Additional keyword arguments used to augment or construct a
775 `DataId`. See `DataId` parameters.
777 Returns
778 -------
779 obj : `DeferredDatasetHandle`
780 A handle which can be used to retrieve a dataset at a later time.
782 Raises
783 ------
784 LookupError
785 Raised if no matching dataset exists in the `Registry` or
786 datastore.
787 ValueError
788 Raised if a resolved `DatasetRef` was passed as an input, but it
789 differs from the one found in the registry.
790 TypeError
791 Raised if no collections were provided.
792 """
793 raise NotImplementedError()
795 @abstractmethod
796 def get(
797 self,
798 datasetRefOrType: DatasetRef | DatasetType | str,
799 /,
800 dataId: DataId | None = None,
801 *,
802 parameters: dict[str, Any] | None = None,
803 collections: Any = None,
804 storageClass: StorageClass | str | None = None,
805 timespan: Timespan | None = None,
806 **kwargs: Any,
807 ) -> Any:
808 """Retrieve a stored dataset.
810 Parameters
811 ----------
812 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
813 When `DatasetRef` the `dataId` should be `None`.
814 Otherwise the `DatasetType` or name thereof.
815 If a resolved `DatasetRef`, the associated dataset
816 is returned directly without additional querying.
817 dataId : `dict` or `DataCoordinate`
818 A `dict` of `Dimension` link name, value pairs that label the
819 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
820 should be provided as the first argument.
821 parameters : `dict`
822 Additional StorageClass-defined options to control reading,
823 typically used to efficiently read only a subset of the dataset.
824 collections : Any, optional
825 Collections to be searched, overriding ``self.collections``.
826 Can be any of the types supported by the ``collections`` argument
827 to butler construction.
828 storageClass : `StorageClass` or `str`, optional
829 The storage class to be used to override the Python type
830 returned by this method. By default the returned type matches
831 the dataset type definition for this dataset. Specifying a
832 read `StorageClass` can force a different type to be returned.
833 This type must be compatible with the original type.
834 timespan : `Timespan` or `None`, optional
835 A timespan that the validity range of the dataset must overlap.
836 If not provided and this is a calibration dataset type, an attempt
837 will be made to find the timespan from any temporal coordinate
838 in the data ID.
839 **kwargs
840 Additional keyword arguments used to augment or construct a
841 `DataCoordinate`. See `DataCoordinate.standardize`
842 parameters.
844 Returns
845 -------
846 obj : `object`
847 The dataset.
849 Raises
850 ------
851 LookupError
852 Raised if no matching dataset exists in the `Registry`.
853 TypeError
854 Raised if no collections were provided.
856 Notes
857 -----
858 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
859 this method requires that the given data ID include temporal dimensions
860 beyond the dimensions of the dataset type itself, in order to find the
861 dataset with the appropriate validity range. For example, a "bias"
862 dataset with native dimensions ``{instrument, detector}`` could be
863 fetched with a ``{instrument, detector, exposure}`` data ID, because
864 ``exposure`` is a temporal dimension.
865 """
866 raise NotImplementedError()
868 @abstractmethod
869 def getURIs(
870 self,
871 datasetRefOrType: DatasetRef | DatasetType | str,
872 /,
873 dataId: DataId | None = None,
874 *,
875 predict: bool = False,
876 collections: Any = None,
877 run: str | None = None,
878 **kwargs: Any,
879 ) -> DatasetRefURIs:
880 """Return the URIs associated with the dataset.
882 Parameters
883 ----------
884 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
885 When `DatasetRef` the `dataId` should be `None`.
886 Otherwise the `DatasetType` or name thereof.
887 dataId : `dict` or `DataCoordinate`
888 A `dict` of `Dimension` link name, value pairs that label the
889 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
890 should be provided as the first argument.
891 predict : `bool`
892 If `True`, allow URIs to be returned of datasets that have not
893 been written.
894 collections : Any, optional
895 Collections to be searched, overriding ``self.collections``.
896 Can be any of the types supported by the ``collections`` argument
897 to butler construction.
898 run : `str`, optional
899 Run to use for predictions, overriding ``self.run``.
900 **kwargs
901 Additional keyword arguments used to augment or construct a
902 `DataCoordinate`. See `DataCoordinate.standardize`
903 parameters.
905 Returns
906 -------
907 uris : `DatasetRefURIs`
908 The URI to the primary artifact associated with this dataset (if
909 the dataset was disassembled within the datastore this may be
910 `None`), and the URIs to any components associated with the dataset
911 artifact. (can be empty if there are no components).
912 """
913 raise NotImplementedError()
915 def getURI(
916 self,
917 datasetRefOrType: DatasetRef | DatasetType | str,
918 /,
919 dataId: DataId | None = None,
920 *,
921 predict: bool = False,
922 collections: Any = None,
923 run: str | None = None,
924 **kwargs: Any,
925 ) -> ResourcePath:
926 """Return the URI to the Dataset.
928 Parameters
929 ----------
930 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
931 When `DatasetRef` the `dataId` should be `None`.
932 Otherwise the `DatasetType` or name thereof.
933 dataId : `dict` or `DataCoordinate`
934 A `dict` of `Dimension` link name, value pairs that label the
935 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
936 should be provided as the first argument.
937 predict : `bool`
938 If `True`, allow URIs to be returned of datasets that have not
939 been written.
940 collections : Any, optional
941 Collections to be searched, overriding ``self.collections``.
942 Can be any of the types supported by the ``collections`` argument
943 to butler construction.
944 run : `str`, optional
945 Run to use for predictions, overriding ``self.run``.
946 **kwargs
947 Additional keyword arguments used to augment or construct a
948 `DataCoordinate`. See `DataCoordinate.standardize`
949 parameters.
951 Returns
952 -------
953 uri : `lsst.resources.ResourcePath`
954 URI pointing to the Dataset within the datastore. If the
955 Dataset does not exist in the datastore, and if ``predict`` is
956 `True`, the URI will be a prediction and will include a URI
957 fragment "#predicted".
958 If the datastore does not have entities that relate well
959 to the concept of a URI the returned URI string will be
960 descriptive. The returned URI is not guaranteed to be obtainable.
962 Raises
963 ------
964 LookupError
965 A URI has been requested for a dataset that does not exist and
966 guessing is not allowed.
967 ValueError
968 Raised if a resolved `DatasetRef` was passed as an input, but it
969 differs from the one found in the registry.
970 TypeError
971 Raised if no collections were provided.
972 RuntimeError
973 Raised if a URI is requested for a dataset that consists of
974 multiple artifacts.
975 """
976 primary, components = self.getURIs(
977 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
978 )
980 if primary is None or components:
981 raise RuntimeError(
982 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
983 "Use Butler.getURIs() instead."
984 )
985 return primary
987 @abstractmethod
988 def get_dataset_type(self, name: str) -> DatasetType:
989 """Get the `DatasetType`.
991 Parameters
992 ----------
993 name : `str`
994 Name of the type.
996 Returns
997 -------
998 type : `DatasetType`
999 The `DatasetType` associated with the given name.
1001 Raises
1002 ------
1003 lsst.daf.butler.MissingDatasetTypeError
1004 Raised if the requested dataset type has not been registered.
1006 Notes
1007 -----
1008 This method handles component dataset types automatically, though most
1009 other operations do not.
1010 """
1011 raise NotImplementedError()
1013 @abstractmethod
1014 def get_dataset(
1015 self,
1016 id: DatasetId | str,
1017 *,
1018 storage_class: str | StorageClass | None = None,
1019 dimension_records: bool = False,
1020 datastore_records: bool = False,
1021 ) -> DatasetRef | None:
1022 """Retrieve a Dataset entry.
1024 Parameters
1025 ----------
1026 id : `DatasetId`
1027 The unique identifier for the dataset, as an instance of
1028 `uuid.UUID` or a string containing a hexadecimal number.
1029 storage_class : `str` or `StorageClass` or `None`
1030 A storage class to use when creating the returned entry. If given
1031 it must be compatible with the default storage class.
1032 dimension_records : `bool`, optional
1033 If `True` the ref will be expanded and contain dimension records.
1034 datastore_records : `bool`, optional
1035 If `True` the ref will contain associated datastore records.
1037 Returns
1038 -------
1039 ref : `DatasetRef` or `None`
1040 A ref to the Dataset, or `None` if no matching Dataset
1041 was found.
1042 """
1043 raise NotImplementedError()
1045 @abstractmethod
1046 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1047 """Retrieve a list of dataset entries.
1049 Parameters
1050 ----------
1051 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1052 The unique identifiers for the datasets, as instances of
1053 `uuid.UUID` or strings containing a hexadecimal number.
1055 Returns
1056 -------
1057 refs : `list` [ `DatasetRef` ]
1058 A list containing a `DatasetRef` for each of the given dataset IDs.
1059 If a dataset was not found, no error is thrown -- it is just not
1060 included in the list. The returned datasets are in no particular
1061 order.
1062 """
1063 raise NotImplementedError()
1065 @abstractmethod
1066 def find_dataset(
1067 self,
1068 dataset_type: DatasetType | str,
1069 data_id: DataId | None = None,
1070 *,
1071 collections: str | Sequence[str] | None = None,
1072 timespan: Timespan | None = None,
1073 storage_class: str | StorageClass | None = None,
1074 dimension_records: bool = False,
1075 datastore_records: bool = False,
1076 **kwargs: Any,
1077 ) -> DatasetRef | None:
1078 """Find a dataset given its `DatasetType` and data ID.
1080 This can be used to obtain a `DatasetRef` that permits the dataset to
1081 be read from a `Datastore`. If the dataset is a component and can not
1082 be found using the provided dataset type, a dataset ref for the parent
1083 will be returned instead but with the correct dataset type.
1085 Parameters
1086 ----------
1087 dataset_type : `DatasetType` or `str`
1088 A `DatasetType` or the name of one. If this is a `DatasetType`
1089 instance, its storage class will be respected and propagated to
1090 the output, even if it differs from the dataset type definition
1091 in the registry, as long as the storage classes are convertible.
1092 data_id : `dict` or `DataCoordinate`, optional
1093 A `dict`-like object containing the `Dimension` links that identify
1094 the dataset within a collection. If it is a `dict` the dataId
1095 can include dimension record values such as ``day_obs`` and
1096 ``seq_num`` or ``full_name`` that can be used to derive the
1097 primary dimension.
1098 collections : `str` or `list` [`str`], optional
1099 A an ordered list of collections to search for the dataset.
1100 Defaults to ``self.defaults.collections``.
1101 timespan : `Timespan`, optional
1102 A timespan that the validity range of the dataset must overlap.
1103 If not provided, any `~CollectionType.CALIBRATION` collections
1104 matched by the ``collections`` argument will not be searched.
1105 storage_class : `str` or `StorageClass` or `None`
1106 A storage class to use when creating the returned entry. If given
1107 it must be compatible with the default storage class.
1108 dimension_records : `bool`, optional
1109 If `True` the ref will be expanded and contain dimension records.
1110 datastore_records : `bool`, optional
1111 If `True` the ref will contain associated datastore records.
1112 **kwargs
1113 Additional keyword arguments passed to
1114 `DataCoordinate.standardize` to convert ``dataId`` to a true
1115 `DataCoordinate` or augment an existing one. This can also include
1116 dimension record metadata that can be used to derive a primary
1117 dimension value.
1119 Returns
1120 -------
1121 ref : `DatasetRef`
1122 A reference to the dataset, or `None` if no matching Dataset
1123 was found.
1125 Raises
1126 ------
1127 lsst.daf.butler.NoDefaultCollectionError
1128 Raised if ``collections`` is `None` and
1129 ``self.collections`` is `None`.
1130 LookupError
1131 Raised if one or more data ID keys are missing.
1132 lsst.daf.butler.MissingDatasetTypeError
1133 Raised if the dataset type does not exist.
1134 lsst.daf.butler.MissingCollectionError
1135 Raised if any of ``collections`` does not exist in the registry.
1137 Notes
1138 -----
1139 This method simply returns `None` and does not raise an exception even
1140 when the set of collections searched is intrinsically incompatible with
1141 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1142 only `~CollectionType.CALIBRATION` collections are being searched.
1143 This may make it harder to debug some lookup failures, but the behavior
1144 is intentional; we consider it more important that failed searches are
1145 reported consistently, regardless of the reason, and that adding
1146 additional collections that do not contain a match to the search path
1147 never changes the behavior.
1149 This method handles component dataset types automatically, though most
1150 other query operations do not.
1151 """
1152 raise NotImplementedError()
1154 @abstractmethod
1155 def retrieve_artifacts_zip(
1156 self,
1157 refs: Iterable[DatasetRef],
1158 destination: ResourcePathExpression,
1159 overwrite: bool = True,
1160 ) -> ResourcePath:
1161 """Retrieve artifacts from a Butler and place in ZIP file.
1163 Parameters
1164 ----------
1165 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1166 The datasets to be included in the zip file.
1167 destination : `lsst.resources.ResourcePathExpression`
1168 Directory to write the new ZIP file. This directory will
1169 also be used as a staging area for the datasets being downloaded
1170 from the datastore.
1171 overwrite : `bool`, optional
1172 If `False` the output Zip will not be written if a file of the
1173 same name is already present in ``destination``.
1175 Returns
1176 -------
1177 zip_file : `lsst.resources.ResourcePath`
1178 The path to the new ZIP file.
1180 Raises
1181 ------
1182 ValueError
1183 Raised if there are no refs to retrieve.
1184 """
1185 raise NotImplementedError()
1187 @abstractmethod
1188 def retrieveArtifacts(
1189 self,
1190 refs: Iterable[DatasetRef],
1191 destination: ResourcePathExpression,
1192 transfer: str = "auto",
1193 preserve_path: bool = True,
1194 overwrite: bool = False,
1195 ) -> list[ResourcePath]:
1196 """Retrieve the artifacts associated with the supplied refs.
1198 Parameters
1199 ----------
1200 refs : iterable of `DatasetRef`
1201 The datasets for which artifacts are to be retrieved.
1202 A single ref can result in multiple artifacts. The refs must
1203 be resolved.
1204 destination : `lsst.resources.ResourcePath` or `str`
1205 Location to write the artifacts.
1206 transfer : `str`, optional
1207 Method to use to transfer the artifacts. Must be one of the options
1208 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1209 "move" is not allowed.
1210 preserve_path : `bool`, optional
1211 If `True` the full path of the artifact within the datastore
1212 is preserved. If `False` the final file component of the path
1213 is used.
1214 overwrite : `bool`, optional
1215 If `True` allow transfers to overwrite existing files at the
1216 destination.
1218 Returns
1219 -------
1220 targets : `list` of `lsst.resources.ResourcePath`
1221 URIs of file artifacts in destination location. Order is not
1222 preserved.
1224 Notes
1225 -----
1226 For non-file datastores the artifacts written to the destination
1227 may not match the representation inside the datastore. For example
1228 a hierarchical data structure in a NoSQL database may well be stored
1229 as a JSON file.
1230 """
1231 raise NotImplementedError()
1233 @abstractmethod
1234 def exists(
1235 self,
1236 dataset_ref_or_type: DatasetRef | DatasetType | str,
1237 /,
1238 data_id: DataId | None = None,
1239 *,
1240 full_check: bool = True,
1241 collections: Any = None,
1242 **kwargs: Any,
1243 ) -> DatasetExistence:
1244 """Indicate whether a dataset is known to Butler registry and
1245 datastore.
1247 Parameters
1248 ----------
1249 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1250 When `DatasetRef` the `dataId` should be `None`.
1251 Otherwise the `DatasetType` or name thereof.
1252 data_id : `dict` or `DataCoordinate`
1253 A `dict` of `Dimension` link name, value pairs that label the
1254 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1255 should be provided as the first argument.
1256 full_check : `bool`, optional
1257 If `True`, a check will be made for the actual existence of a
1258 dataset artifact. This will involve additional overhead due to
1259 the need to query an external system. If `False`, this check will
1260 be omitted, and the registry and datastore will solely be asked
1261 if they know about the dataset but no direct check for the
1262 artifact will be performed.
1263 collections : Any, optional
1264 Collections to be searched, overriding ``self.collections``.
1265 Can be any of the types supported by the ``collections`` argument
1266 to butler construction.
1267 **kwargs
1268 Additional keyword arguments used to augment or construct a
1269 `DataCoordinate`. See `DataCoordinate.standardize`
1270 parameters.
1272 Returns
1273 -------
1274 existence : `DatasetExistence`
1275 Object indicating whether the dataset is known to registry and
1276 datastore. Evaluates to `True` if the dataset is present and known
1277 to both.
1278 """
1279 raise NotImplementedError()
1281 @abstractmethod
1282 def _exists_many(
1283 self,
1284 refs: Iterable[DatasetRef],
1285 /,
1286 *,
1287 full_check: bool = True,
1288 ) -> dict[DatasetRef, DatasetExistence]:
1289 """Indicate whether multiple datasets are known to Butler registry and
1290 datastore.
1292 This is an experimental API that may change at any moment.
1294 Parameters
1295 ----------
1296 refs : iterable of `DatasetRef`
1297 The datasets to be checked.
1298 full_check : `bool`, optional
1299 If `True`, a check will be made for the actual existence of each
1300 dataset artifact. This will involve additional overhead due to
1301 the need to query an external system. If `False`, this check will
1302 be omitted, and the registry and datastore will solely be asked
1303 if they know about the dataset(s) but no direct check for the
1304 artifact(s) will be performed.
1306 Returns
1307 -------
1308 existence : dict of [`DatasetRef`, `DatasetExistence`]
1309 Mapping from the given dataset refs to an enum indicating the
1310 status of the dataset in registry and datastore.
1311 Each value evaluates to `True` if the dataset is present and known
1312 to both.
1313 """
1314 raise NotImplementedError()
1316 @abstractmethod
1317 def removeRuns(
1318 self,
1319 names: Iterable[str],
1320 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1321 *,
1322 unlink_from_chains: bool = False,
1323 ) -> None:
1324 """Remove one or more `~CollectionType.RUN` collections and the
1325 datasets within them.
1327 Parameters
1328 ----------
1329 names : `~collections.abc.Iterable` [ `str` ]
1330 The names of the collections to remove.
1331 unstore : `bool`, optional
1332 If `True` (default), delete datasets from all datastores in which
1333 they are present, and attempt to rollback the registry deletions if
1334 datastore deletions fail (which may not always be possible). If
1335 `False`, datastore records for these datasets are still removed,
1336 but any artifacts (e.g. files) will not be. This parameter is now
1337 deprecated and no longer has any effect. Files are always deleted
1338 from datastores unless they were ingested using full URIs.
1339 unlink_from_chains : `bool`, optional
1340 If `True` remove the RUN collection from any chains prior to
1341 removing the RUN. If `False` the removal will fail if any chains
1342 still refer to the RUN.
1344 Raises
1345 ------
1346 TypeError
1347 Raised if one or more collections are not of type
1348 `~CollectionType.RUN`.
1349 """
1350 raise NotImplementedError()
1352 @abstractmethod
1353 def ingest(
1354 self,
1355 *datasets: FileDataset,
1356 transfer: str | None = "auto",
1357 record_validation_info: bool = True,
1358 skip_existing: bool = False,
1359 ) -> None:
1360 """Store and register one or more datasets that already exist on disk.
1362 Parameters
1363 ----------
1364 *datasets : `FileDataset`
1365 Each positional argument is a struct containing information about
1366 a file to be ingested, including its URI (either absolute or
1367 relative to the datastore root, if applicable), a resolved
1368 `DatasetRef`, and optionally a formatter class or its
1369 fully-qualified string name. If a formatter is not provided, the
1370 formatter that would be used for `put` is assumed. On successful
1371 ingest all `FileDataset.formatter` attributes will be set to the
1372 formatter class used. `FileDataset.path` attributes may be modified
1373 to put paths in whatever the datastore considers a standardized
1374 form.
1375 transfer : `str`, optional
1376 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1377 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1378 transfer the file.
1379 record_validation_info : `bool`, optional
1380 If `True`, the default, the datastore can record validation
1381 information associated with the file. If `False` the datastore
1382 will not attempt to track any information such as checksums
1383 or file sizes. This can be useful if such information is tracked
1384 in an external system or if the file is to be compressed in place.
1385 It is up to the datastore whether this parameter is relevant.
1386 skip_existing : `bool`, optional
1387 If `True`, a dataset will not be ingested if a dataset with the
1388 same dataset ID already exists in the datastore.
1389 If `False` (the default), a `ConflictingDefinitionError` will be
1390 raised if any datasets with the same dataset ID already exist
1391 in the datastore.
1393 Raises
1394 ------
1395 TypeError
1396 Raised if the butler is read-only or if no run was provided.
1397 NotImplementedError
1398 Raised if the `Datastore` does not support the given transfer mode.
1399 DatasetTypeNotSupportedError
1400 Raised if one or more files to be ingested have a dataset type that
1401 is not supported by the `Datastore`..
1402 FileNotFoundError
1403 Raised if one of the given files does not exist.
1404 FileExistsError
1405 Raised if transfer is not `None` but the (internal) location the
1406 file would be moved to is already occupied.
1407 ConflictingDefinitionError
1408 Raised if a dataset already exists in the repository and
1409 ``skip_existing`` is `False`.
1411 Notes
1412 -----
1413 This operation is not fully exception safe: if a database operation
1414 fails, the given `FileDataset` instances may be only partially updated.
1416 It is atomic in terms of database operations (they will either all
1417 succeed or all fail) providing the database engine implements
1418 transactions correctly. It will attempt to be atomic in terms of
1419 filesystem operations as well, but this cannot be implemented
1420 rigorously for most datastores.
1421 """
1422 raise NotImplementedError()
1424 @abstractmethod
1425 def ingest_zip(
1426 self,
1427 zip_file: ResourcePathExpression,
1428 transfer: str = "auto",
1429 *,
1430 transfer_dimensions: bool = False,
1431 dry_run: bool = False,
1432 ) -> None:
1433 """Ingest a Zip file into this butler.
1435 The Zip file must have been created by `retrieve_artifacts_zip`.
1437 Parameters
1438 ----------
1439 zip_file : `lsst.resources.ResourcePathExpression`
1440 Path to the Zip file.
1441 transfer : `str`, optional
1442 Method to use to transfer the Zip into the datastore.
1443 transfer_dimensions : `bool`, optional
1444 If `True`, dimension record data associated with the new datasets
1445 will be transferred from the Zip file, if present.
1446 dry_run : `bool`, optional
1447 If `True` the ingest will be processed without any modifications
1448 made to the target butler and as if the target butler did not
1449 have any of the datasets.
1451 Notes
1452 -----
1453 Run collections and dataset types are created as needed.
1454 """
1455 raise NotImplementedError()
1457 @abstractmethod
1458 def export(
1459 self,
1460 *,
1461 directory: str | None = None,
1462 filename: str | None = None,
1463 format: str | None = None,
1464 transfer: str | None = None,
1465 ) -> AbstractContextManager[RepoExportContext]:
1466 """Export datasets from the repository represented by this `Butler`.
1468 This method is a context manager that returns a helper object
1469 (`RepoExportContext`) that is used to indicate what information from
1470 the repository should be exported.
1472 Parameters
1473 ----------
1474 directory : `str`, optional
1475 Directory dataset files should be written to if ``transfer`` is not
1476 `None`.
1477 filename : `str`, optional
1478 Name for the file that will include database information associated
1479 with the exported datasets. If this is not an absolute path and
1480 ``directory`` is not `None`, it will be written to ``directory``
1481 instead of the current working directory. Defaults to
1482 "export.{format}".
1483 format : `str`, optional
1484 File format for the database information file. If `None`, the
1485 extension of ``filename`` will be used.
1486 transfer : `str`, optional
1487 Transfer mode passed to `Datastore.export`.
1489 Raises
1490 ------
1491 TypeError
1492 Raised if the set of arguments passed is inconsistent.
1494 Examples
1495 --------
1496 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1497 methods are used to provide the iterables over data IDs and/or datasets
1498 to be exported::
1500 with butler.export("exports.yaml") as export:
1501 # Export all flats, but none of the dimension element rows
1502 # (i.e. data ID information) associated with them.
1503 export.saveDatasets(
1504 butler.registry.queryDatasets("flat"), elements=()
1505 )
1506 # Export all datasets that start with "deepCoadd_" and all of
1507 # their associated data ID information.
1508 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1509 """
1510 raise NotImplementedError()
1512 @abstractmethod
1513 def import_(
1514 self,
1515 *,
1516 directory: ResourcePathExpression | None = None,
1517 filename: ResourcePathExpression | TextIO | None = None,
1518 format: str | None = None,
1519 transfer: str | None = None,
1520 skip_dimensions: set | None = None,
1521 record_validation_info: bool = True,
1522 without_datastore: bool = False,
1523 ) -> None:
1524 """Import datasets into this repository that were exported from a
1525 different butler repository via `~lsst.daf.butler.Butler.export`.
1527 Parameters
1528 ----------
1529 directory : `~lsst.resources.ResourcePathExpression`, optional
1530 Directory containing dataset files to import from. If `None`,
1531 ``filename`` and all dataset file paths specified therein must
1532 be absolute.
1533 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1534 A stream or name of file that contains database information
1535 associated with the exported datasets, typically generated by
1536 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1537 `~lsst.resources.ResourcePath` and is not an absolute path,
1538 it will first be looked for relative to ``directory`` and if not
1539 found there it will be looked for in the current working
1540 directory. Defaults to "export.{format}".
1541 format : `str`, optional
1542 File format for ``filename``. If `None`, the extension of
1543 ``filename`` will be used.
1544 transfer : `str`, optional
1545 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1546 skip_dimensions : `set`, optional
1547 Names of dimensions that should be skipped and not imported.
1548 record_validation_info : `bool`, optional
1549 If `True`, the default, the datastore can record validation
1550 information associated with the file. If `False` the datastore
1551 will not attempt to track any information such as checksums
1552 or file sizes. This can be useful if such information is tracked
1553 in an external system or if the file is to be compressed in place.
1554 It is up to the datastore whether this parameter is relevant.
1555 without_datastore : `bool`, optional
1556 If `True` only registry records will be imported and the datastore
1557 will be ignored.
1559 Raises
1560 ------
1561 TypeError
1562 Raised if the set of arguments passed is inconsistent, or if the
1563 butler is read-only.
1564 """
1565 raise NotImplementedError()
1567 @abstractmethod
1568 def transfer_dimension_records_from(
1569 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef | DataCoordinate]
1570 ) -> None:
1571 """Transfer dimension records to this Butler from another Butler.
1573 Parameters
1574 ----------
1575 source_butler : `LimitedButler` or `Butler`
1576 Butler from which the records are to be transferred. If data IDs
1577 in ``source_refs`` are not expanded then this has to be a full
1578 `Butler` whose registry will be used to expand data IDs. If the
1579 source refs contain coordinates that are used to populate other
1580 records then this will also need to be a full `Butler`.
1581 source_refs : iterable of `DatasetRef` or `DataCoordinate`
1582 Datasets or data IDs defined in the source butler whose dimension
1583 records should be transferred to this butler.
1584 """
1585 raise NotImplementedError()
1587 @abstractmethod
1588 def transfer_from(
1589 self,
1590 source_butler: LimitedButler,
1591 source_refs: Iterable[DatasetRef],
1592 transfer: str = "auto",
1593 skip_missing: bool = True,
1594 register_dataset_types: bool = False,
1595 transfer_dimensions: bool = False,
1596 dry_run: bool = False,
1597 ) -> Collection[DatasetRef]:
1598 """Transfer datasets to this Butler from a run in another Butler.
1600 Parameters
1601 ----------
1602 source_butler : `LimitedButler`
1603 Butler from which the datasets are to be transferred. If data IDs
1604 in ``source_refs`` are not expanded then this has to be a full
1605 `Butler` whose registry will be used to expand data IDs.
1606 source_refs : iterable of `DatasetRef`
1607 Datasets defined in the source butler that should be transferred to
1608 this butler. In most circumstances, ``transfer_from`` is faster if
1609 the dataset refs are expanded.
1610 transfer : `str`, optional
1611 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1612 skip_missing : `bool`
1613 If `True`, datasets with no datastore artifact associated with
1614 them are not transferred. If `False` a registry entry will be
1615 created even if no datastore record is created (and so will
1616 look equivalent to the dataset being unstored).
1617 register_dataset_types : `bool`
1618 If `True` any missing dataset types are registered. Otherwise
1619 an exception is raised.
1620 transfer_dimensions : `bool`, optional
1621 If `True`, dimension record data associated with the new datasets
1622 will be transferred.
1623 dry_run : `bool`, optional
1624 If `True` the transfer will be processed without any modifications
1625 made to the target butler and as if the target butler did not
1626 have any of the datasets.
1628 Returns
1629 -------
1630 refs : `list` of `DatasetRef`
1631 The refs added to this Butler.
1633 Notes
1634 -----
1635 The datastore artifact has to exist for a transfer
1636 to be made but non-existence is not an error.
1638 Datasets that already exist in this run will be skipped.
1640 The datasets are imported as part of a transaction, although
1641 dataset types are registered before the transaction is started.
1642 This means that it is possible for a dataset type to be registered
1643 even though transfer has failed.
1644 """
1645 raise NotImplementedError()
1647 @abstractmethod
1648 def validateConfiguration(
1649 self,
1650 logFailures: bool = False,
1651 datasetTypeNames: Iterable[str] | None = None,
1652 ignore: Iterable[str] | None = None,
1653 ) -> None:
1654 """Validate butler configuration.
1656 Checks that each `DatasetType` can be stored in the `Datastore`.
1658 Parameters
1659 ----------
1660 logFailures : `bool`, optional
1661 If `True`, output a log message for every validation error
1662 detected.
1663 datasetTypeNames : iterable of `str`, optional
1664 The `DatasetType` names that should be checked. This allows
1665 only a subset to be selected.
1666 ignore : iterable of `str`, optional
1667 Names of DatasetTypes to skip over. This can be used to skip
1668 known problems. If a named `DatasetType` corresponds to a
1669 composite, all components of that `DatasetType` will also be
1670 ignored.
1672 Raises
1673 ------
1674 ButlerValidationError
1675 Raised if there is some inconsistency with how this Butler
1676 is configured.
1677 """
1678 raise NotImplementedError()
1680 @property
1681 @abstractmethod
1682 def collection_chains(self) -> ButlerCollections:
1683 """Object with methods for modifying collection chains
1684 (`~lsst.daf.butler.ButlerCollections`).
1686 Deprecated. Replaced with ``collections`` property.
1687 """
1688 raise NotImplementedError()
1690 @property
1691 @abstractmethod
1692 def collections(self) -> ButlerCollections:
1693 """Object with methods for modifying and querying collections
1694 (`~lsst.daf.butler.ButlerCollections`).
1696 Use of this object is preferred over `registry` wherever possible.
1697 """
1698 raise NotImplementedError()
1700 @property
1701 @abstractmethod
1702 def run(self) -> str | None:
1703 """Name of the run this butler writes outputs to by default (`str` or
1704 `None`).
1705 """
1706 raise NotImplementedError()
1708 @property
1709 @abstractmethod
1710 def registry(self) -> Registry:
1711 """The object that manages dataset metadata and relationships
1712 (`Registry`).
1714 Many operations that don't involve reading or writing butler datasets
1715 are accessible only via `Registry` methods. Eventually these methods
1716 will be replaced by equivalent `Butler` methods.
1717 """
1718 raise NotImplementedError()
1720 @abstractmethod
1721 def query(self) -> AbstractContextManager[Query]:
1722 """Context manager returning a `Query` object used for construction
1723 and execution of complex queries.
1724 """
1725 raise NotImplementedError()
1727 def query_data_ids(
1728 self,
1729 dimensions: DimensionGroup | Iterable[str] | str,
1730 *,
1731 data_id: DataId | None = None,
1732 where: str = "",
1733 bind: Mapping[str, Any] | None = None,
1734 with_dimension_records: bool = False,
1735 order_by: Iterable[str] | str | None = None,
1736 limit: int | None = -20_000,
1737 explain: bool = True,
1738 **kwargs: Any,
1739 ) -> list[DataCoordinate]:
1740 """Query for data IDs matching user-provided criteria.
1742 Parameters
1743 ----------
1744 dimensions : `DimensionGroup`, `str`, or \
1745 `~collections.abc.Iterable` [`str`]
1746 The dimensions of the data IDs to yield, as either `DimensionGroup`
1747 instances or `str`. Will be automatically expanded to a complete
1748 `DimensionGroup`.
1749 data_id : `dict` or `DataCoordinate`, optional
1750 A data ID whose key-value pairs are used as equality constraints
1751 in the query.
1752 where : `str`, optional
1753 A string expression similar to a SQL WHERE clause. May involve
1754 any column of a dimension table or (as a shortcut for the primary
1755 key column of a dimension table) dimension name. See
1756 :ref:`daf_butler_dimension_expressions` for more information.
1757 bind : `~collections.abc.Mapping`, optional
1758 Mapping containing literal values that should be injected into the
1759 ``where`` expression, keyed by the identifiers they replace.
1760 Values of collection type can be expanded in some cases; see
1761 :ref:`daf_butler_dimension_expressions_identifiers` for more
1762 information.
1763 with_dimension_records : `bool`, optional
1764 If `True` (default is `False`) then returned data IDs will have
1765 dimension records.
1766 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1767 Names of the columns/dimensions to use for ordering returned data
1768 IDs. Column name can be prefixed with minus (``-``) to use
1769 descending ordering.
1770 limit : `int` or `None`, optional
1771 Upper limit on the number of returned records. `None` can be used
1772 if no limit is wanted. A limit of ``0`` means that the query will
1773 be executed and validated but no results will be returned. In this
1774 case there will be no exception even if ``explain`` is `True`.
1775 If a negative value is given a warning will be issued if the number
1776 of results is capped by that limit.
1777 explain : `bool`, optional
1778 If `True` (default) then `EmptyQueryResultError` exception is
1779 raised when resulting list is empty. The exception contains
1780 non-empty list of strings explaining possible causes for empty
1781 result.
1782 **kwargs
1783 Additional keyword arguments are forwarded to
1784 `DataCoordinate.standardize` when processing the ``data_id``
1785 argument (and may be used to provide a constraining data ID even
1786 when the ``data_id`` argument is `None`).
1788 Returns
1789 -------
1790 dataIds : `list` [`DataCoordinate`]
1791 Data IDs matching the given query parameters. These are always
1792 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1793 returns `True`).
1795 Raises
1796 ------
1797 lsst.daf.butler.registry.DataIdError
1798 Raised when ``data_id`` or keyword arguments specify unknown
1799 dimensions or values, or when they contain inconsistent values.
1800 lsst.daf.butler.registry.UserExpressionError
1801 Raised when ``where`` expression is invalid.
1802 lsst.daf.butler.EmptyQueryResultError
1803 Raised when query generates empty result and ``explain`` is set to
1804 `True`.
1805 TypeError
1806 Raised when the arguments are incompatible.
1807 """
1808 if data_id is None:
1809 data_id = DataCoordinate.make_empty(self.dimensions)
1810 if order_by is None:
1811 order_by = []
1812 query_limit = limit
1813 warn_limit = False
1814 if limit is not None and limit < 0:
1815 query_limit = abs(limit) + 1
1816 warn_limit = True
1817 with self.query() as query:
1818 result = (
1819 query.data_ids(dimensions)
1820 .where(data_id, where, bind=bind, **kwargs)
1821 .order_by(*ensure_iterable(order_by))
1822 .limit(query_limit)
1823 )
1824 if with_dimension_records:
1825 result = result.with_dimension_records()
1826 data_ids = list(result)
1827 if warn_limit and len(data_ids) == query_limit:
1828 # We asked for one too many so must remove that from the list.
1829 data_ids.pop(-1)
1830 assert limit is not None # For mypy.
1831 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1832 if explain and (limit is None or limit != 0) and not data_ids:
1833 raise EmptyQueryResultError(list(result.explain_no_results()))
1834 return data_ids
1836 def query_datasets(
1837 self,
1838 dataset_type: str | DatasetType,
1839 collections: str | Iterable[str] | None = None,
1840 *,
1841 find_first: bool = True,
1842 data_id: DataId | None = None,
1843 where: str = "",
1844 bind: Mapping[str, Any] | None = None,
1845 with_dimension_records: bool = False,
1846 order_by: Iterable[str] | str | None = None,
1847 limit: int | None = -20_000,
1848 explain: bool = True,
1849 **kwargs: Any,
1850 ) -> list[DatasetRef]:
1851 """Query for dataset references matching user-provided criteria.
1853 Parameters
1854 ----------
1855 dataset_type : `str` or `DatasetType`
1856 Dataset type object or name to search for.
1857 collections : collection expression, optional
1858 A collection name or iterable of collection names to search. If not
1859 provided, the default collections are used. Can be a wildcard if
1860 ``find_first`` is `False` (if find first is requested the order
1861 of collections matters and wildcards make the order indeterminate).
1862 See :ref:`daf_butler_collection_expressions` for more information.
1863 find_first : `bool`, optional
1864 If `True` (default), for each result data ID, only yield one
1865 `DatasetRef` of each `DatasetType`, from the first collection in
1866 which a dataset of that dataset type appears (according to the
1867 order of ``collections`` passed in). If `True`, ``collections``
1868 must not contain wildcards.
1869 data_id : `dict` or `DataCoordinate`, optional
1870 A data ID whose key-value pairs are used as equality constraints in
1871 the query.
1872 where : `str`, optional
1873 A string expression similar to a SQL WHERE clause. May involve any
1874 column of a dimension table or (as a shortcut for the primary key
1875 column of a dimension table) dimension name. See
1876 :ref:`daf_butler_dimension_expressions` for more information.
1877 bind : `~collections.abc.Mapping`, optional
1878 Mapping containing literal values that should be injected into the
1879 ``where`` expression, keyed by the identifiers they replace. Values
1880 of collection type can be expanded in some cases; see
1881 :ref:`daf_butler_dimension_expressions_identifiers` for more
1882 information.
1883 with_dimension_records : `bool`, optional
1884 If `True` (default is `False`) then returned data IDs will have
1885 dimension records.
1886 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1887 Names of the columns/dimensions to use for ordering returned data
1888 IDs. Column name can be prefixed with minus (``-``) to use
1889 descending ordering.
1890 limit : `int` or `None`, optional
1891 Upper limit on the number of returned records. `None` can be used
1892 if no limit is wanted. A limit of ``0`` means that the query will
1893 be executed and validated but no results will be returned. In this
1894 case there will be no exception even if ``explain`` is `True`.
1895 If a negative value is given a warning will be issued if the number
1896 of results is capped by that limit.
1897 explain : `bool`, optional
1898 If `True` (default) then `EmptyQueryResultError` exception is
1899 raised when resulting list is empty. The exception contains
1900 non-empty list of strings explaining possible causes for empty
1901 result.
1902 **kwargs
1903 Additional keyword arguments are forwarded to
1904 `DataCoordinate.standardize` when processing the ``data_id``
1905 argument (and may be used to provide a constraining data ID even
1906 when the ``data_id`` argument is `None`).
1908 Returns
1909 -------
1910 refs : `.queries.DatasetRefQueryResults`
1911 Dataset references matching the given query criteria. Nested data
1912 IDs are guaranteed to include values for all implied dimensions
1913 (i.e. `DataCoordinate.hasFull` will return `True`).
1915 Raises
1916 ------
1917 lsst.daf.butler.registry.DatasetTypeExpressionError
1918 Raised when ``dataset_type`` expression is invalid.
1919 lsst.daf.butler.registry.DataIdError
1920 Raised when ``data_id`` or keyword arguments specify unknown
1921 dimensions or values, or when they contain inconsistent values.
1922 lsst.daf.butler.registry.UserExpressionError
1923 Raised when ``where`` expression is invalid.
1924 lsst.daf.butler.EmptyQueryResultError
1925 Raised when query generates empty result and ``explain`` is set to
1926 `True`.
1927 TypeError
1928 Raised when the arguments are incompatible, such as when a
1929 collection wildcard is passed when ``find_first`` is `True`, or
1930 when ``collections`` is `None` and default butler collections are
1931 not defined.
1932 """
1933 if data_id is None:
1934 data_id = DataCoordinate.make_empty(self.dimensions)
1935 if order_by is None:
1936 order_by = []
1937 if collections and has_globs(collections):
1938 # Wild cards need to be expanded but can only be allowed if
1939 # find_first=False because expanding wildcards does not return
1940 # a guaranteed ordering. Querying collection registry to expand
1941 # collections when we do not have wildcards is expensive so only
1942 # do it if we need it.
1943 if find_first:
1944 raise InvalidQueryError(
1945 f"Can not use wildcards in collections when find_first=True (given {collections})"
1946 )
1947 collections = self.collections.query(collections)
1948 query_limit = limit
1949 warn_limit = False
1950 if limit is not None and limit < 0:
1951 query_limit = abs(limit) + 1
1952 warn_limit = True
1953 with self.query() as query:
1954 result = (
1955 query.datasets(dataset_type, collections=collections, find_first=find_first)
1956 .where(data_id, where, bind=bind, **kwargs)
1957 .order_by(*ensure_iterable(order_by))
1958 .limit(query_limit)
1959 )
1960 if with_dimension_records:
1961 result = result.with_dimension_records()
1962 refs = list(result)
1963 if warn_limit and len(refs) == query_limit:
1964 # We asked for one too many so must remove that from the list.
1965 refs.pop(-1)
1966 assert limit is not None # For mypy.
1967 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1968 if explain and (limit is None or limit != 0) and not refs:
1969 raise EmptyQueryResultError(list(result.explain_no_results()))
1970 return refs
1972 def query_dimension_records(
1973 self,
1974 element: str,
1975 *,
1976 data_id: DataId | None = None,
1977 where: str = "",
1978 bind: Mapping[str, Any] | None = None,
1979 order_by: Iterable[str] | str | None = None,
1980 limit: int | None = -20_000,
1981 explain: bool = True,
1982 **kwargs: Any,
1983 ) -> list[DimensionRecord]:
1984 """Query for dimension information matching user-provided criteria.
1986 Parameters
1987 ----------
1988 element : `str`
1989 The name of a dimension element to obtain records for.
1990 data_id : `dict` or `DataCoordinate`, optional
1991 A data ID whose key-value pairs are used as equality constraints
1992 in the query.
1993 where : `str`, optional
1994 A string expression similar to a SQL WHERE clause. See
1995 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1996 information.
1997 bind : `~collections.abc.Mapping`, optional
1998 Mapping containing literal values that should be injected into the
1999 ``where`` expression, keyed by the identifiers they replace.
2000 Values of collection type can be expanded in some cases; see
2001 :ref:`daf_butler_dimension_expressions_identifiers` for more
2002 information.
2003 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2004 Names of the columns/dimensions to use for ordering returned data
2005 IDs. Column name can be prefixed with minus (``-``) to use
2006 descending ordering.
2007 limit : `int` or `None`, optional
2008 Upper limit on the number of returned records. `None` can be used
2009 if no limit is wanted. A limit of ``0`` means that the query will
2010 be executed and validated but no results will be returned. In this
2011 case there will be no exception even if ``explain`` is `True`.
2012 If a negative value is given a warning will be issued if the number
2013 of results is capped by that limit.
2014 explain : `bool`, optional
2015 If `True` (default) then `EmptyQueryResultError` exception is
2016 raised when resulting list is empty. The exception contains
2017 non-empty list of strings explaining possible causes for empty
2018 result.
2019 **kwargs
2020 Additional keyword arguments are forwarded to
2021 `DataCoordinate.standardize` when processing the ``data_id``
2022 argument (and may be used to provide a constraining data ID even
2023 when the ``data_id`` argument is `None`).
2025 Returns
2026 -------
2027 records : `list`[`DimensionRecord`]
2028 Dimension records matching the given query parameters.
2030 Raises
2031 ------
2032 lsst.daf.butler.registry.DataIdError
2033 Raised when ``data_id`` or keyword arguments specify unknown
2034 dimensions or values, or when they contain inconsistent values.
2035 lsst.daf.butler.registry.UserExpressionError
2036 Raised when ``where`` expression is invalid.
2037 lsst.daf.butler.EmptyQueryResultError
2038 Raised when query generates empty result and ``explain`` is set to
2039 `True`.
2040 TypeError
2041 Raised when the arguments are incompatible, such as when a
2042 collection wildcard is passed when ``find_first`` is `True`, or
2043 when ``collections`` is `None` and default butler collections are
2044 not defined.
2045 """
2046 if data_id is None:
2047 data_id = DataCoordinate.make_empty(self.dimensions)
2048 if order_by is None:
2049 order_by = []
2050 query_limit = limit
2051 warn_limit = False
2052 if limit is not None and limit < 0:
2053 query_limit = abs(limit) + 1
2054 warn_limit = True
2055 with self.query() as query:
2056 result = (
2057 query.dimension_records(element)
2058 .where(data_id, where, bind=bind, **kwargs)
2059 .order_by(*ensure_iterable(order_by))
2060 .limit(query_limit)
2061 )
2062 dimension_records = list(result)
2063 if warn_limit and len(dimension_records) == query_limit:
2064 # We asked for one too many so must remove that from the list.
2065 dimension_records.pop(-1)
2066 assert limit is not None # For mypy.
2067 _LOG.warning(
2068 "More dimension records are available than the requested limit of %d.", abs(limit)
2069 )
2070 if explain and (limit is None or limit != 0) and not dimension_records:
2071 raise EmptyQueryResultError(list(result.explain_no_results()))
2072 return dimension_records
2074 def query_all_datasets(
2075 self,
2076 collections: str | Iterable[str] | None = None,
2077 *,
2078 name: str | Iterable[str] = "*",
2079 find_first: bool = True,
2080 data_id: DataId | None = None,
2081 where: str = "",
2082 bind: Mapping[str, Any] | None = None,
2083 limit: int | None = -20_000,
2084 **kwargs: Any,
2085 ) -> list[DatasetRef]:
2086 """Query for datasets of potentially multiple types.
2088 Parameters
2089 ----------
2090 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2091 The collection or collections to search, in order. If not provided
2092 or `None`, the default collection search path for this butler is
2093 used.
2094 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2095 Names or name patterns (glob-style) that returned dataset type
2096 names must match. If an iterable, items are OR'd together. The
2097 default is to include all dataset types in the given collections.
2098 find_first : `bool`, optional
2099 If `True` (default), for each result data ID, only yield one
2100 `DatasetRef` of each `DatasetType`, from the first collection in
2101 which a dataset of that dataset type appears (according to the
2102 order of ``collections`` passed in).
2103 data_id : `dict` or `DataCoordinate`, optional
2104 A data ID whose key-value pairs are used as equality constraints in
2105 the query.
2106 where : `str`, optional
2107 A string expression similar to a SQL WHERE clause. May involve any
2108 column of a dimension table or (as a shortcut for the primary key
2109 column of a dimension table) dimension name. See
2110 :ref:`daf_butler_dimension_expressions` for more information.
2111 bind : `~collections.abc.Mapping`, optional
2112 Mapping containing literal values that should be injected into the
2113 ``where`` expression, keyed by the identifiers they replace. Values
2114 of collection type can be expanded in some cases; see
2115 :ref:`daf_butler_dimension_expressions_identifiers` for more
2116 information.
2117 limit : `int` or `None`, optional
2118 Upper limit on the number of returned records. `None` can be used
2119 if no limit is wanted. A limit of ``0`` means that the query will
2120 be executed and validated but no results will be returned.
2121 If a negative value is given a warning will be issued if the number
2122 of results is capped by that limit. If no limit is provided, by
2123 default a maximum of 20,000 records will be returned.
2124 **kwargs
2125 Additional keyword arguments are forwarded to
2126 `DataCoordinate.standardize` when processing the ``data_id``
2127 argument (and may be used to provide a constraining data ID even
2128 when the ``data_id`` argument is `None`).
2130 Raises
2131 ------
2132 MissingDatasetTypeError
2133 When no dataset types match ``name``, or an explicit (non-glob)
2134 dataset type in ``name`` does not exist.
2135 InvalidQueryError
2136 If the parameters to the query are inconsistent or malformed.
2137 MissingCollectionError
2138 If a given collection is not found.
2140 Returns
2141 -------
2142 refs : `list` [ `DatasetRef` ]
2143 Dataset references matching the given query criteria. Nested data
2144 IDs are guaranteed to include values for all implied dimensions
2145 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2146 include dimension records (`DataCoordinate.hasRecords` will be
2147 `False`).
2148 """
2149 if collections is None:
2150 collections = list(self.collections.defaults)
2151 else:
2152 collections = list(ensure_iterable(collections))
2154 if bind is None:
2155 bind = {}
2156 if data_id is None:
2157 data_id = {}
2159 warn_limit = False
2160 if limit is not None and limit < 0:
2161 # Add one to the limit so we can detect if we have exceeded it.
2162 limit = abs(limit) + 1
2163 warn_limit = True
2165 args = QueryAllDatasetsParameters(
2166 collections=collections,
2167 name=list(ensure_iterable(name)),
2168 find_first=find_first,
2169 data_id=data_id,
2170 where=where,
2171 limit=limit,
2172 bind=bind,
2173 kwargs=kwargs,
2174 with_dimension_records=False,
2175 )
2176 with self._query_all_datasets_by_page(args) as pages:
2177 result = []
2178 for page in pages:
2179 result.extend(page)
2181 if warn_limit and limit is not None and len(result) >= limit:
2182 # Remove the extra dataset we added for the limit check.
2183 result.pop()
2184 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2186 return result
2188 @abstractmethod
2189 def _query_all_datasets_by_page(
2190 self, args: QueryAllDatasetsParameters
2191 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2192 raise NotImplementedError()
2194 def clone(
2195 self,
2196 *,
2197 collections: CollectionArgType | None | EllipsisType = ...,
2198 run: str | None | EllipsisType = ...,
2199 inferDefaults: bool | EllipsisType = ...,
2200 dataId: dict[str, str] | EllipsisType = ...,
2201 metrics: ButlerMetrics | None = None,
2202 ) -> Butler:
2203 """Return a new Butler instance connected to the same repository
2204 as this one, optionally overriding ``collections``, ``run``,
2205 ``inferDefaults``, and default data ID.
2207 Parameters
2208 ----------
2209 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2210 optional
2211 Same as constructor. If omitted, uses value from original object.
2212 run : `str` or `None`, optional
2213 Same as constructor. If `None`, no default run is used. If
2214 omitted, copies value from original object.
2215 inferDefaults : `bool`, optional
2216 Same as constructor. If omitted, copies value from original
2217 object.
2218 dataId : `str`
2219 Same as ``kwargs`` passed to the constructor. If omitted, copies
2220 values from original object.
2221 metrics : `ButlerMetrics` or `None`, optional
2222 Metrics object to record butler statistics.
2223 """
2224 raise NotImplementedError()
2226 @abstractmethod
2227 def close(self) -> None:
2228 raise NotImplementedError()
2230 @abstractmethod
2231 def _expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
2232 raise NotImplementedError()