Coverage for python / lsst / daf / butler / _butler.py: 30%
316 statements
« prev ^ index » next coverage.py v7.13.2, created at 2026-02-08 16:37 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["Butler"]
32import dataclasses
33import urllib.parse
34import uuid
35import warnings
36from abc import abstractmethod
37from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
38from contextlib import AbstractContextManager
39from types import EllipsisType
40from typing import TYPE_CHECKING, Any, TextIO
42from lsst.resources import ResourcePath, ResourcePathExpression
43from lsst.utils import doImportType
44from lsst.utils.iteration import ensure_iterable
45from lsst.utils.logging import getLogger
47from ._butler_collections import ButlerCollections
48from ._butler_config import ButlerConfig, ButlerType
49from ._butler_instance_options import ButlerInstanceOptions
50from ._butler_metrics import ButlerMetrics
51from ._butler_repo_index import ButlerRepoIndex
52from ._config import Config, ConfigSubset
53from ._exceptions import EmptyQueryResultError, InvalidQueryError
54from ._limited_butler import LimitedButler
55from ._query_all_datasets import QueryAllDatasetsParameters
56from .datastore import Datastore
57from .dimensions import DataCoordinate, DimensionConfig
58from .registry import RegistryConfig, _RegistryFactory
59from .repo_relocation import BUTLER_ROOT_TAG
60from .utils import has_globs
62if TYPE_CHECKING:
63 from ._dataset_existence import DatasetExistence
64 from ._dataset_provenance import DatasetProvenance
65 from ._dataset_ref import DatasetId, DatasetRef
66 from ._dataset_type import DatasetType
67 from ._deferredDatasetHandle import DeferredDatasetHandle
68 from ._file_dataset import FileDataset
69 from ._labeled_butler_factory import LabeledButlerFactoryProtocol
70 from ._storage_class import StorageClass
71 from ._timespan import Timespan
72 from .datastore import DatasetRefURIs
73 from .dimensions import DataId, DimensionGroup, DimensionRecord
74 from .queries import Query
75 from .registry import CollectionArgType, Registry
76 from .transfers import RepoExportContext
78_LOG = getLogger(__name__)
81@dataclasses.dataclass
82class ParsedButlerDatasetURI:
83 label: str
84 dataset_id: uuid.UUID
85 uri: str
88@dataclasses.dataclass
89class SpecificButlerDataset:
90 butler: Butler
91 dataset: DatasetRef | None
94class _DeprecatedDefault:
95 """Default value for a deprecated parameter."""
98class Butler(LimitedButler): # numpydoc ignore=PR02
99 """Interface for data butler and factory for Butler instances.
101 Parameters
102 ----------
103 config : `ButlerConfig`, `Config` or `str`, optional
104 Configuration. Anything acceptable to the `ButlerConfig` constructor.
105 If a directory path is given the configuration will be read from a
106 ``butler.yaml`` file in that location. If `None` is given default
107 values will be used. If ``config`` contains "cls" key then its value is
108 used as a name of butler class and it must be a sub-class of this
109 class, otherwise `DirectButler` is instantiated.
110 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
111 An expression specifying the collections to be searched (in order) when
112 reading datasets.
113 This may be a `str` collection name or an iterable thereof.
114 See :ref:`daf_butler_collection_expressions` for more information.
115 These collections are not registered automatically and must be
116 manually registered before they are used by any method, but they may be
117 manually registered after the `Butler` is initialized.
118 run : `str`, optional
119 Name of the `~CollectionType.RUN` collection new datasets should be
120 inserted into. If ``collections`` is `None` and ``run`` is not `None`,
121 ``collections`` will be set to ``[run]``. If not `None`, this
122 collection will automatically be registered. If this is not set (and
123 ``writeable`` is not set either), a read-only butler will be created.
124 searchPaths : `list` of `str`, optional
125 Directory paths to search when calculating the full Butler
126 configuration. Not used if the supplied config is already a
127 `ButlerConfig`.
128 writeable : `bool`, optional
129 Explicitly sets whether the butler supports write operations. If not
130 provided, a read-write butler is created if any of ``run``, ``tags``,
131 or ``chains`` is non-empty.
132 inferDefaults : `bool`, optional
133 If `True` (default) infer default data ID values from the values
134 present in the datasets in ``collections``: if all collections have the
135 same value (or no value) for a governor dimension, that value will be
136 the default for that dimension. Nonexistent collections are ignored.
137 If a default value is provided explicitly for a governor dimension via
138 ``**kwargs``, no default will be inferred for that dimension.
139 without_datastore : `bool`, optional
140 If `True` do not attach a datastore to this butler. Any attempts
141 to use a datastore will fail.
142 metrics : `ButlerMetrics` or `None`
143 External metrics object to be used for tracking butler usage. If `None`
144 a new metrics object is created.
145 **kwargs : `typing.Any`
146 Additional keyword arguments passed to a constructor of actual butler
147 class.
149 Notes
150 -----
151 The preferred way to instantiate Butler is via the `from_config` method.
152 The call to ``Butler(...)`` is equivalent to ``Butler.from_config(...)``,
153 but ``mypy`` will complain about the former.
154 """
156 def __new__(
157 cls,
158 config: Config | ResourcePathExpression | None = None,
159 *,
160 collections: Any = None,
161 run: str | None = None,
162 searchPaths: Sequence[ResourcePathExpression] | None = None,
163 writeable: bool | None = None,
164 inferDefaults: bool = True,
165 without_datastore: bool = False,
166 metrics: ButlerMetrics | None = None,
167 **kwargs: Any,
168 ) -> Butler:
169 if cls is Butler:
170 return Butler.from_config(
171 config=config,
172 collections=collections,
173 run=run,
174 searchPaths=searchPaths,
175 writeable=writeable,
176 inferDefaults=inferDefaults,
177 without_datastore=without_datastore,
178 metrics=metrics,
179 **kwargs,
180 )
182 # Note: we do not pass any parameters to __new__, Python will pass them
183 # to __init__ after __new__ returns sub-class instance.
184 return super().__new__(cls)
186 @classmethod
187 def from_config(
188 cls,
189 config: Config | ResourcePathExpression | None = None,
190 *,
191 collections: Any = None,
192 run: str | None = None,
193 searchPaths: Sequence[ResourcePathExpression] | None = None,
194 writeable: bool | None = None,
195 inferDefaults: bool = True,
196 without_datastore: bool = False,
197 metrics: ButlerMetrics | None = None,
198 **kwargs: Any,
199 ) -> Butler:
200 """Create butler instance from configuration.
202 Parameters
203 ----------
204 config : `ButlerConfig`, `Config` or `str`, optional
205 Configuration. Anything acceptable to the `ButlerConfig`
206 constructor. If a directory path is given the configuration will be
207 read from a ``butler.yaml`` file in that location. If `None` is
208 given default values will be used. If ``config`` contains "cls" key
209 then its value is used as a name of butler class and it must be a
210 sub-class of this class, otherwise `DirectButler` is instantiated.
211 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
212 An expression specifying the collections to be searched (in order)
213 when reading datasets.
214 This may be a `str` collection name or an iterable thereof.
215 See :ref:`daf_butler_collection_expressions` for more information.
216 These collections are not registered automatically and must be
217 manually registered before they are used by any method, but they
218 may be manually registered after the `Butler` is initialized.
219 run : `str`, optional
220 Name of the `~CollectionType.RUN` collection new datasets should be
221 inserted into. If ``collections`` is `None` and ``run`` is not
222 `None`, ``collections`` will be set to ``[run]``. If not `None`,
223 this collection will automatically be registered. If this is not
224 set (and ``writeable`` is not set either), a read-only butler will
225 be created.
226 searchPaths : `list` of `str`, optional
227 Directory paths to search when calculating the full Butler
228 configuration. Not used if the supplied config is already a
229 `ButlerConfig`.
230 writeable : `bool`, optional
231 Explicitly sets whether the butler supports write operations. If
232 not provided, a read-write butler is created if any of ``run``,
233 ``tags``, or ``chains`` is non-empty.
234 inferDefaults : `bool`, optional
235 If `True` (default) infer default data ID values from the values
236 present in the datasets in ``collections``: if all collections have
237 the same value (or no value) for a governor dimension, that value
238 will be the default for that dimension. Nonexistent collections
239 are ignored. If a default value is provided explicitly for a
240 governor dimension via ``**kwargs``, no default will be inferred
241 for that dimension.
242 without_datastore : `bool`, optional
243 If `True` do not attach a datastore to this butler. Any attempts
244 to use a datastore will fail.
245 metrics : `ButlerMetrics` or `None`, optional
246 Metrics object to record butler usage statistics.
247 **kwargs : `typing.Any`
248 Default data ID key-value pairs. These may only identify
249 "governor" dimensions like ``instrument`` and ``skymap``.
251 Returns
252 -------
253 butler : `Butler`
254 A `Butler` constructed from the given configuration.
256 Notes
257 -----
258 Calling this factory method is identical to calling
259 ``Butler(config, ...)``. Its only raison d'être is that ``mypy``
260 complains about ``Butler()`` call.
262 Examples
263 --------
264 While there are many ways to control exactly how a `Butler` interacts
265 with the collections in its `Registry`, the most common cases are still
266 simple.
268 For a read-only `Butler` that searches one collection, do::
270 butler = Butler.from_config(
271 "/path/to/repo", collections=["u/alice/DM-50000"]
272 )
274 For a read-write `Butler` that writes to and reads from a
275 `~CollectionType.RUN` collection::
277 butler = Butler.from_config(
278 "/path/to/repo", run="u/alice/DM-50000/a"
279 )
281 The `Butler` passed to a ``PipelineTask`` is often much more complex,
282 because we want to write to one `~CollectionType.RUN` collection but
283 read from several others (as well)::
285 butler = Butler.from_config(
286 "/path/to/repo",
287 run="u/alice/DM-50000/a",
288 collections=[
289 "u/alice/DM-50000/a",
290 "u/bob/DM-49998",
291 "HSC/defaults",
292 ],
293 )
295 This butler will `put` new datasets to the run ``u/alice/DM-50000/a``.
296 Datasets will be read first from that run (since it appears first in
297 the chain), and then from ``u/bob/DM-49998`` and finally
298 ``HSC/defaults``.
300 Finally, one can always create a `Butler` with no collections::
302 butler = Butler.from_config("/path/to/repo", writeable=True)
304 This can be extremely useful when you just want to use
305 ``butler.registry``, e.g. for inserting dimension data or managing
306 collections, or when the collections you want to use with the butler
307 are not consistent. Passing ``writeable`` explicitly here is only
308 necessary if you want to be able to make changes to the repo - usually
309 the value for ``writeable`` can be guessed from the collection
310 arguments provided, but it defaults to `False` when there are not
311 collection arguments.
312 """
313 # DirectButler used to have a way to specify a "copy constructor" by
314 # passing the "butler" parameter to its constructor. This has
315 # been moved out of the constructor into Butler.clone().
316 butler = kwargs.pop("butler", None)
317 metrics = metrics if metrics is not None else ButlerMetrics()
318 if butler is not None:
319 if not isinstance(butler, Butler):
320 raise TypeError("'butler' parameter must be a Butler instance")
321 if config is not None or searchPaths is not None or writeable is not None:
322 raise TypeError(
323 "Cannot pass 'config', 'searchPaths', or 'writeable' arguments with 'butler' argument."
324 )
325 return butler.clone(
326 collections=collections, run=run, inferDefaults=inferDefaults, metrics=metrics, dataId=kwargs
327 )
329 options = ButlerInstanceOptions(
330 collections=collections,
331 run=run,
332 writeable=writeable,
333 inferDefaults=inferDefaults,
334 metrics=metrics,
335 kwargs=kwargs,
336 )
338 # Load the Butler configuration. This may involve searching the
339 # environment to locate a configuration file.
340 butler_config = ButlerConfig(config, searchPaths=searchPaths, without_datastore=without_datastore)
341 butler_type = butler_config.get_butler_type()
343 # Make DirectButler if class is not specified.
344 match butler_type:
345 case ButlerType.DIRECT:
346 from .direct_butler import DirectButler
348 return DirectButler.create_from_config(
349 butler_config,
350 options=options,
351 without_datastore=without_datastore,
352 )
353 case ButlerType.REMOTE:
354 from .remote_butler._factory import RemoteButlerFactory
356 # Assume this is being created by a client who would like
357 # default caching of remote datasets.
358 factory = RemoteButlerFactory.create_factory_from_config(butler_config)
359 return factory.create_butler_with_credentials_from_environment(
360 butler_options=options, enable_datastore_cache=True
361 )
362 case _:
363 raise TypeError(f"Unknown Butler type '{butler_type}'")
365 @staticmethod
366 def makeRepo(
367 root: ResourcePathExpression,
368 config: Config | str | None = None,
369 dimensionConfig: Config | str | None = None,
370 standalone: bool = False,
371 searchPaths: list[str] | None = None,
372 forceConfigRoot: bool = True,
373 outfile: ResourcePathExpression | None = None,
374 overwrite: bool = False,
375 ) -> Config:
376 """Create an empty data repository by adding a butler.yaml config
377 to a repository root directory.
379 Parameters
380 ----------
381 root : `lsst.resources.ResourcePathExpression`
382 Path or URI to the root location of the new repository. Will be
383 created if it does not exist.
384 config : `Config` or `str`, optional
385 Configuration to write to the repository, after setting any
386 root-dependent Registry or Datastore config options. Can not
387 be a `ButlerConfig` or a `ConfigSubset`. If `None`, default
388 configuration will be used. Root-dependent config options
389 specified in this config are overwritten if ``forceConfigRoot``
390 is `True`.
391 dimensionConfig : `Config` or `str`, optional
392 Configuration for dimensions, will be used to initialize registry
393 database.
394 standalone : `bool`
395 If True, write all expanded defaults, not just customized or
396 repository-specific settings.
397 This (mostly) decouples the repository from the default
398 configuration, insulating it from changes to the defaults (which
399 may be good or bad, depending on the nature of the changes).
400 Future *additions* to the defaults will still be picked up when
401 initializing `Butlers` to repos created with ``standalone=True``.
402 searchPaths : `list` of `str`, optional
403 Directory paths to search when calculating the full butler
404 configuration.
405 forceConfigRoot : `bool`, optional
406 If `False`, any values present in the supplied ``config`` that
407 would normally be reset are not overridden and will appear
408 directly in the output config. This allows non-standard overrides
409 of the root directory for a datastore or registry to be given.
410 If this parameter is `True` the values for ``root`` will be
411 forced into the resulting config if appropriate.
412 outfile : `lss.resources.ResourcePathExpression`, optional
413 If not-`None`, the output configuration will be written to this
414 location rather than into the repository itself. Can be a URI
415 string. Can refer to a directory that will be used to write
416 ``butler.yaml``.
417 overwrite : `bool`, optional
418 Create a new configuration file even if one already exists
419 in the specified output location. Default is to raise
420 an exception.
422 Returns
423 -------
424 config : `Config`
425 The updated `Config` instance written to the repo.
427 Raises
428 ------
429 ValueError
430 Raised if a ButlerConfig or ConfigSubset is passed instead of a
431 regular Config (as these subclasses would make it impossible to
432 support ``standalone=False``).
433 FileExistsError
434 Raised if the output config file already exists.
435 os.error
436 Raised if the directory does not exist, exists but is not a
437 directory, or cannot be created.
439 Notes
440 -----
441 Note that when ``standalone=False`` (the default), the configuration
442 search path (see `ConfigSubset.defaultSearchPaths`) that was used to
443 construct the repository should also be used to construct any Butlers
444 to avoid configuration inconsistencies.
445 """
446 if isinstance(config, ButlerConfig | ConfigSubset):
447 raise ValueError("makeRepo must be passed a regular Config without defaults applied.")
449 # Ensure that the root of the repository exists or can be made
450 root_uri = ResourcePath(root, forceDirectory=True)
451 root_uri.mkdir()
453 config = Config(config)
455 # If we are creating a new repo from scratch with relative roots,
456 # do not propagate an explicit root from the config file
457 if "root" in config:
458 del config["root"]
460 full = ButlerConfig(config, searchPaths=searchPaths) # this applies defaults
461 imported_class = doImportType(full["datastore", "cls"])
462 if not issubclass(imported_class, Datastore):
463 raise TypeError(f"Imported datastore class {full['datastore', 'cls']} is not a Datastore")
464 datastoreClass: type[Datastore] = imported_class
465 datastoreClass.setConfigRoot(BUTLER_ROOT_TAG, config, full, overwrite=forceConfigRoot)
467 # if key exists in given config, parse it, otherwise parse the defaults
468 # in the expanded config
469 if config.get(("registry", "db")):
470 registryConfig = RegistryConfig(config)
471 else:
472 registryConfig = RegistryConfig(full)
473 defaultDatabaseUri = registryConfig.makeDefaultDatabaseUri(BUTLER_ROOT_TAG)
474 if defaultDatabaseUri is not None:
475 Config.updateParameters(
476 RegistryConfig, config, full, toUpdate={"db": defaultDatabaseUri}, overwrite=forceConfigRoot
477 )
478 else:
479 Config.updateParameters(RegistryConfig, config, full, toCopy=("db",), overwrite=forceConfigRoot)
481 if standalone:
482 config.merge(full)
483 else:
484 # Always expand the registry.managers section into the per-repo
485 # config, because after the database schema is created, it's not
486 # allowed to change anymore. Note that in the standalone=True
487 # branch, _everything_ in the config is expanded, so there's no
488 # need to special case this.
489 Config.updateParameters(RegistryConfig, config, full, toMerge=("managers",), overwrite=False)
490 configURI: ResourcePathExpression
491 if outfile is not None:
492 # When writing to a separate location we must include
493 # the root of the butler repo in the config else it won't know
494 # where to look.
495 config["root"] = root_uri.geturl()
496 configURI = outfile
497 else:
498 configURI = root_uri
499 # Check that if obscore key is present then its config must be there
500 # too, this is to avoid common mistake when people copy butler.yaml
501 # from existing repo with obscore but do not fill its config.
502 if (obscore_key := ("registry", "managers", "obscore")) in config:
503 obscore_config_key = ("registry", "managers", "obscore", "config")
504 if obscore_config_key not in config or not config[obscore_config_key]:
505 warnings.warn(
506 "Obscore manager is declared in registry configuration, "
507 "but obscore configuration is missing, obscore manager will be removed.",
508 stacklevel=2,
509 )
510 del config[obscore_key]
511 # Strip obscore configuration, if it is present, before writing config
512 # to a file, obscore config will be stored in registry.
513 if (obscore_config_key := ("registry", "managers", "obscore", "config")) in config:
514 config_to_write = config.copy()
515 del config_to_write[obscore_config_key]
516 config_to_write.dumpToUri(configURI, overwrite=overwrite)
517 # configFile attribute is updated, need to copy it to original.
518 config.configFile = config_to_write.configFile
519 else:
520 config.dumpToUri(configURI, overwrite=overwrite)
522 # Create Registry and populate tables
523 registryConfig = RegistryConfig(config.get("registry"))
524 dimensionConfig = DimensionConfig(dimensionConfig)
525 registry = _RegistryFactory(registryConfig).create_from_config(
526 dimensionConfig=dimensionConfig, butlerRoot=root_uri
527 )
528 registry.close()
530 _LOG.verbose("Wrote new Butler configuration file to %s", configURI)
532 return config
534 @classmethod
535 def get_repo_uri(cls, label: str, return_label: bool = False) -> ResourcePath:
536 """Look up the label in a butler repository index.
538 Parameters
539 ----------
540 label : `str`
541 Label of the Butler repository to look up.
542 return_label : `bool`, optional
543 If ``label`` cannot be found in the repository index (either
544 because index is not defined or ``label`` is not in the index) and
545 ``return_label`` is `True` then return ``ResourcePath(label)``.
546 If ``return_label`` is `False` (default) then an exception will be
547 raised instead.
549 Returns
550 -------
551 uri : `lsst.resources.ResourcePath`
552 URI to the Butler repository associated with the given label or
553 default value if it is provided.
555 Raises
556 ------
557 KeyError
558 Raised if the label is not found in the index, or if an index
559 is not defined, and ``return_label`` is `False`.
561 Notes
562 -----
563 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
564 information is discovered.
565 """
566 return ButlerRepoIndex.get_repo_uri(label, return_label)
568 @classmethod
569 def get_known_repos(cls) -> set[str]:
570 """Retrieve the list of known repository labels.
572 Returns
573 -------
574 repos : `set` of `str`
575 All the known labels. Can be empty if no index can be found.
577 Notes
578 -----
579 See `~lsst.daf.butler.ButlerRepoIndex` for details on how the
580 information is discovered.
581 """
582 return ButlerRepoIndex.get_known_repos()
584 @classmethod
585 def parse_dataset_uri(cls, uri: str) -> ParsedButlerDatasetURI:
586 """Extract the butler label and dataset ID from a dataset URI.
588 Parameters
589 ----------
590 uri : `str`
591 The dataset URI to parse.
593 Returns
594 -------
595 parsed : `ParsedButlerDatasetURI`
596 The label associated with the butler repository from which this
597 dataset originates and the ID of the dataset.
599 Notes
600 -----
601 Supports dataset URIs of the forms
602 ``ivo://org.rubinobs/usdac/dr1?repo=butler_label&id=UUID`` (see
603 DMTN-302) and ``butler://butler_label/UUID``. The ``butler`` URI is
604 deprecated and can not include ``/`` in the label string. ``ivo`` URIs
605 can include anything supported by the `Butler` constructor, including
606 paths to repositories and alias labels.
608 ivo://org.rubinobs/dr1?repo=/repo/main&id=UUID
610 will return a label of ``/repo/main``.
612 This method does not attempt to check that the dataset exists in the
613 labeled butler.
615 Since the IVOID can be issued by any publisher to represent a Butler
616 dataset there is no validation of the path or netloc component of the
617 URI. The only requirement is that there are ``id`` and ``repo`` keys
618 in the ``ivo`` URI query component.
619 """
620 parsed = urllib.parse.urlparse(uri)
621 parsed_scheme = parsed.scheme.lower()
622 if parsed_scheme == "ivo":
623 # Do not validate the netloc or the path values.
624 qs = urllib.parse.parse_qs(parsed.query)
625 if "repo" not in qs or "id" not in qs:
626 raise ValueError(f"Missing 'repo' and/or 'id' query parameters in IVOID {uri}.")
627 if len(qs["repo"]) != 1 or len(qs["id"]) != 1:
628 raise ValueError(f"Butler IVOID only supports a single value of repo and id, got {uri}")
629 label = qs["repo"][0]
630 id_ = qs["id"][0]
631 elif parsed_scheme == "butler":
632 label = parsed.netloc # Butler label is case sensitive.
633 # Need to strip the leading /.
634 id_ = parsed.path[1:]
635 else:
636 raise ValueError(f"Unrecognized URI scheme: {uri!r}")
637 # Strip trailing/leading whitespace from label.
638 label = label.strip()
639 if not label:
640 raise ValueError(f"No butler repository label found in uri {uri!r}")
641 try:
642 dataset_id = uuid.UUID(hex=id_)
643 except Exception as e:
644 e.add_note(f"Error extracting dataset ID from uri {uri!r} with dataset ID string {id_!r}")
645 raise
647 return ParsedButlerDatasetURI(label=label, dataset_id=dataset_id, uri=uri)
649 @classmethod
650 def get_dataset_from_uri(
651 cls, uri: str, factory: LabeledButlerFactoryProtocol | None = None
652 ) -> SpecificButlerDataset:
653 """Get the dataset associated with the given dataset URI.
655 Parameters
656 ----------
657 uri : `str`
658 The URI associated with a dataset.
659 factory : `LabeledButlerFactoryProtocol` or `None`, optional
660 Bound factory function that will be given the butler label
661 and receive a `Butler`. If this is not provided the label
662 will be tried directly.
664 Returns
665 -------
666 result : `SpecificButlerDataset`
667 The butler associated with this URI and the dataset itself.
668 The dataset can be `None` if the UUID is valid but the dataset
669 is not known to this butler.
670 """
671 parsed = cls.parse_dataset_uri(uri)
672 butler: Butler | None = None
673 if factory is not None:
674 # If the label is not recognized, it might be a path.
675 try:
676 butler = factory(parsed.label)
677 except KeyError:
678 pass
679 if butler is None:
680 butler = cls.from_config(parsed.label)
681 return SpecificButlerDataset(butler=butler, dataset=butler.get_dataset(parsed.dataset_id))
683 @abstractmethod
684 def _caching_context(self) -> AbstractContextManager[None]:
685 """Context manager that enables caching."""
686 raise NotImplementedError()
688 @abstractmethod
689 def transaction(self) -> AbstractContextManager[None]:
690 """Context manager supporting `Butler` transactions.
692 Transactions can be nested.
693 """
694 raise NotImplementedError()
696 @abstractmethod
697 def put(
698 self,
699 obj: Any,
700 datasetRefOrType: DatasetRef | DatasetType | str,
701 /,
702 dataId: DataId | None = None,
703 *,
704 run: str | None = None,
705 provenance: DatasetProvenance | None = None,
706 **kwargs: Any,
707 ) -> DatasetRef:
708 """Store and register a dataset.
710 Parameters
711 ----------
712 obj : `object`
713 The dataset.
714 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
715 When `DatasetRef` is provided, ``dataId`` should be `None`.
716 Otherwise the `DatasetType` or name thereof. If a fully resolved
717 `DatasetRef` is given the run and ID are used directly.
718 dataId : `dict` or `DataCoordinate`
719 A `dict` of `Dimension` link name, value pairs that label the
720 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
721 should be provided as the second argument.
722 run : `str`, optional
723 The name of the run the dataset should be added to, overriding
724 ``self.run``. Not used if a resolved `DatasetRef` is provided.
725 provenance : `DatasetProvenance` or `None`, optional
726 Any provenance that should be attached to the serialized dataset.
727 Not supported by all serialization mechanisms.
728 **kwargs
729 Additional keyword arguments used to augment or construct a
730 `DataCoordinate`. See `DataCoordinate.standardize`
731 parameters. Not used if a resolve `DatasetRef` is provided.
733 Returns
734 -------
735 ref : `DatasetRef`
736 A reference to the stored dataset, updated with the correct id if
737 given.
739 Raises
740 ------
741 TypeError
742 Raised if the butler is read-only or if no run has been provided.
743 """
744 raise NotImplementedError()
746 @abstractmethod
747 def getDeferred(
748 self,
749 datasetRefOrType: DatasetRef | DatasetType | str,
750 /,
751 dataId: DataId | None = None,
752 *,
753 parameters: dict | None = None,
754 collections: Any = None,
755 storageClass: str | StorageClass | None = None,
756 timespan: Timespan | None = None,
757 **kwargs: Any,
758 ) -> DeferredDatasetHandle:
759 """Create a `DeferredDatasetHandle` which can later retrieve a dataset,
760 after an immediate registry lookup.
762 Parameters
763 ----------
764 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
765 When `DatasetRef` the `dataId` should be `None`.
766 Otherwise the `DatasetType` or name thereof.
767 dataId : `dict` or `DataCoordinate`, optional
768 A `dict` of `Dimension` link name, value pairs that label the
769 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
770 should be provided as the first argument.
771 parameters : `dict`
772 Additional StorageClass-defined options to control reading,
773 typically used to efficiently read only a subset of the dataset.
774 collections : Any, optional
775 Collections to be searched, overriding ``self.collections``.
776 Can be any of the types supported by the ``collections`` argument
777 to butler construction.
778 storageClass : `StorageClass` or `str`, optional
779 The storage class to be used to override the Python type
780 returned by this method. By default the returned type matches
781 the dataset type definition for this dataset. Specifying a
782 read `StorageClass` can force a different type to be returned.
783 This type must be compatible with the original type.
784 timespan : `Timespan` or `None`, optional
785 A timespan that the validity range of the dataset must overlap.
786 If not provided and this is a calibration dataset type, an attempt
787 will be made to find the timespan from any temporal coordinate
788 in the data ID.
789 **kwargs
790 Additional keyword arguments used to augment or construct a
791 `DataId`. See `DataId` parameters.
793 Returns
794 -------
795 obj : `DeferredDatasetHandle`
796 A handle which can be used to retrieve a dataset at a later time.
798 Raises
799 ------
800 LookupError
801 Raised if no matching dataset exists in the `Registry` or
802 datastore.
803 ValueError
804 Raised if a resolved `DatasetRef` was passed as an input, but it
805 differs from the one found in the registry.
806 TypeError
807 Raised if no collections were provided.
808 """
809 raise NotImplementedError()
811 @abstractmethod
812 def get(
813 self,
814 datasetRefOrType: DatasetRef | DatasetType | str,
815 /,
816 dataId: DataId | None = None,
817 *,
818 parameters: dict[str, Any] | None = None,
819 collections: Any = None,
820 storageClass: StorageClass | str | None = None,
821 timespan: Timespan | None = None,
822 **kwargs: Any,
823 ) -> Any:
824 """Retrieve a stored dataset.
826 Parameters
827 ----------
828 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
829 When `DatasetRef` the `dataId` should be `None`.
830 Otherwise the `DatasetType` or name thereof.
831 If a resolved `DatasetRef`, the associated dataset
832 is returned directly without additional querying.
833 dataId : `dict` or `DataCoordinate`
834 A `dict` of `Dimension` link name, value pairs that label the
835 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
836 should be provided as the first argument.
837 parameters : `dict`
838 Additional StorageClass-defined options to control reading,
839 typically used to efficiently read only a subset of the dataset.
840 collections : Any, optional
841 Collections to be searched, overriding ``self.collections``.
842 Can be any of the types supported by the ``collections`` argument
843 to butler construction.
844 storageClass : `StorageClass` or `str`, optional
845 The storage class to be used to override the Python type
846 returned by this method. By default the returned type matches
847 the dataset type definition for this dataset. Specifying a
848 read `StorageClass` can force a different type to be returned.
849 This type must be compatible with the original type.
850 timespan : `Timespan` or `None`, optional
851 A timespan that the validity range of the dataset must overlap.
852 If not provided and this is a calibration dataset type, an attempt
853 will be made to find the timespan from any temporal coordinate
854 in the data ID.
855 **kwargs
856 Additional keyword arguments used to augment or construct a
857 `DataCoordinate`. See `DataCoordinate.standardize`
858 parameters.
860 Returns
861 -------
862 obj : `object`
863 The dataset.
865 Raises
866 ------
867 LookupError
868 Raised if no matching dataset exists in the `Registry`.
869 TypeError
870 Raised if no collections were provided.
872 Notes
873 -----
874 When looking up datasets in a `~CollectionType.CALIBRATION` collection,
875 this method requires that the given data ID include temporal dimensions
876 beyond the dimensions of the dataset type itself, in order to find the
877 dataset with the appropriate validity range. For example, a "bias"
878 dataset with native dimensions ``{instrument, detector}`` could be
879 fetched with a ``{instrument, detector, exposure}`` data ID, because
880 ``exposure`` is a temporal dimension.
881 """
882 raise NotImplementedError()
884 @abstractmethod
885 def getURIs(
886 self,
887 datasetRefOrType: DatasetRef | DatasetType | str,
888 /,
889 dataId: DataId | None = None,
890 *,
891 predict: bool = False,
892 collections: Any = None,
893 run: str | None = None,
894 **kwargs: Any,
895 ) -> DatasetRefURIs:
896 """Return the URIs associated with the dataset.
898 Parameters
899 ----------
900 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
901 When `DatasetRef` the `dataId` should be `None`.
902 Otherwise the `DatasetType` or name thereof.
903 dataId : `dict` or `DataCoordinate`
904 A `dict` of `Dimension` link name, value pairs that label the
905 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
906 should be provided as the first argument.
907 predict : `bool`
908 If `True`, allow URIs to be returned of datasets that have not
909 been written.
910 collections : Any, optional
911 Collections to be searched, overriding ``self.collections``.
912 Can be any of the types supported by the ``collections`` argument
913 to butler construction.
914 run : `str`, optional
915 Run to use for predictions, overriding ``self.run``.
916 **kwargs
917 Additional keyword arguments used to augment or construct a
918 `DataCoordinate`. See `DataCoordinate.standardize`
919 parameters.
921 Returns
922 -------
923 uris : `DatasetRefURIs`
924 The URI to the primary artifact associated with this dataset (if
925 the dataset was disassembled within the datastore this may be
926 `None`), and the URIs to any components associated with the dataset
927 artifact. (can be empty if there are no components).
928 """
929 raise NotImplementedError()
931 def getURI(
932 self,
933 datasetRefOrType: DatasetRef | DatasetType | str,
934 /,
935 dataId: DataId | None = None,
936 *,
937 predict: bool = False,
938 collections: Any = None,
939 run: str | None = None,
940 **kwargs: Any,
941 ) -> ResourcePath:
942 """Return the URI to the Dataset.
944 Parameters
945 ----------
946 datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
947 When `DatasetRef` the `dataId` should be `None`.
948 Otherwise the `DatasetType` or name thereof.
949 dataId : `dict` or `DataCoordinate`
950 A `dict` of `Dimension` link name, value pairs that label the
951 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
952 should be provided as the first argument.
953 predict : `bool`
954 If `True`, allow URIs to be returned of datasets that have not
955 been written.
956 collections : Any, optional
957 Collections to be searched, overriding ``self.collections``.
958 Can be any of the types supported by the ``collections`` argument
959 to butler construction.
960 run : `str`, optional
961 Run to use for predictions, overriding ``self.run``.
962 **kwargs
963 Additional keyword arguments used to augment or construct a
964 `DataCoordinate`. See `DataCoordinate.standardize`
965 parameters.
967 Returns
968 -------
969 uri : `lsst.resources.ResourcePath`
970 URI pointing to the Dataset within the datastore. If the
971 Dataset does not exist in the datastore, and if ``predict`` is
972 `True`, the URI will be a prediction and will include a URI
973 fragment "#predicted".
974 If the datastore does not have entities that relate well
975 to the concept of a URI the returned URI string will be
976 descriptive. The returned URI is not guaranteed to be obtainable.
978 Raises
979 ------
980 LookupError
981 A URI has been requested for a dataset that does not exist and
982 guessing is not allowed.
983 ValueError
984 Raised if a resolved `DatasetRef` was passed as an input, but it
985 differs from the one found in the registry.
986 TypeError
987 Raised if no collections were provided.
988 RuntimeError
989 Raised if a URI is requested for a dataset that consists of
990 multiple artifacts.
991 """
992 primary, components = self.getURIs(
993 datasetRefOrType, dataId=dataId, predict=predict, collections=collections, run=run, **kwargs
994 )
996 if primary is None or components:
997 raise RuntimeError(
998 f"Dataset ({datasetRefOrType}) includes distinct URIs for components. "
999 "Use Butler.getURIs() instead."
1000 )
1001 return primary
1003 @abstractmethod
1004 def get_dataset_type(self, name: str) -> DatasetType:
1005 """Get the `DatasetType`.
1007 Parameters
1008 ----------
1009 name : `str`
1010 Name of the type.
1012 Returns
1013 -------
1014 type : `DatasetType`
1015 The `DatasetType` associated with the given name.
1017 Raises
1018 ------
1019 lsst.daf.butler.MissingDatasetTypeError
1020 Raised if the requested dataset type has not been registered.
1022 Notes
1023 -----
1024 This method handles component dataset types automatically, though most
1025 other operations do not.
1026 """
1027 raise NotImplementedError()
1029 @abstractmethod
1030 def get_dataset(
1031 self,
1032 id: DatasetId | str,
1033 *,
1034 storage_class: str | StorageClass | None = None,
1035 dimension_records: bool = False,
1036 datastore_records: bool = False,
1037 ) -> DatasetRef | None:
1038 """Retrieve a Dataset entry.
1040 Parameters
1041 ----------
1042 id : `DatasetId`
1043 The unique identifier for the dataset, as an instance of
1044 `uuid.UUID` or a string containing a hexadecimal number.
1045 storage_class : `str` or `StorageClass` or `None`
1046 A storage class to use when creating the returned entry. If given
1047 it must be compatible with the default storage class.
1048 dimension_records : `bool`, optional
1049 If `True` the ref will be expanded and contain dimension records.
1050 datastore_records : `bool`, optional
1051 If `True` the ref will contain associated datastore records.
1053 Returns
1054 -------
1055 ref : `DatasetRef` or `None`
1056 A ref to the Dataset, or `None` if no matching Dataset
1057 was found.
1058 """
1059 raise NotImplementedError()
1061 @abstractmethod
1062 def get_many_datasets(self, ids: Iterable[DatasetId | str]) -> list[DatasetRef]:
1063 """Retrieve a list of dataset entries.
1065 Parameters
1066 ----------
1067 ids : `~collections.abc.Iterable` [ `DatasetId` or `str` ]
1068 The unique identifiers for the datasets, as instances of
1069 `uuid.UUID` or strings containing a hexadecimal number.
1071 Returns
1072 -------
1073 refs : `list` [ `DatasetRef` ]
1074 A list containing a `DatasetRef` for each of the given dataset IDs.
1075 If a dataset was not found, no error is thrown -- it is just not
1076 included in the list. The returned datasets are in no particular
1077 order.
1078 """
1079 raise NotImplementedError()
1081 @abstractmethod
1082 def find_dataset(
1083 self,
1084 dataset_type: DatasetType | str,
1085 data_id: DataId | None = None,
1086 *,
1087 collections: str | Sequence[str] | None = None,
1088 timespan: Timespan | None = None,
1089 storage_class: str | StorageClass | None = None,
1090 dimension_records: bool = False,
1091 datastore_records: bool = False,
1092 **kwargs: Any,
1093 ) -> DatasetRef | None:
1094 """Find a dataset given its `DatasetType` and data ID.
1096 This can be used to obtain a `DatasetRef` that permits the dataset to
1097 be read from a `Datastore`. If the dataset is a component and can not
1098 be found using the provided dataset type, a dataset ref for the parent
1099 will be returned instead but with the correct dataset type.
1101 Parameters
1102 ----------
1103 dataset_type : `DatasetType` or `str`
1104 A `DatasetType` or the name of one. If this is a `DatasetType`
1105 instance, its storage class will be respected and propagated to
1106 the output, even if it differs from the dataset type definition
1107 in the registry, as long as the storage classes are convertible.
1108 data_id : `dict` or `DataCoordinate`, optional
1109 A `dict`-like object containing the `Dimension` links that identify
1110 the dataset within a collection. If it is a `dict` the dataId
1111 can include dimension record values such as ``day_obs`` and
1112 ``seq_num`` or ``full_name`` that can be used to derive the
1113 primary dimension.
1114 collections : `str` or `list` [`str`], optional
1115 A an ordered list of collections to search for the dataset.
1116 Defaults to ``self.defaults.collections``.
1117 timespan : `Timespan`, optional
1118 A timespan that the validity range of the dataset must overlap.
1119 If not provided, any `~CollectionType.CALIBRATION` collections
1120 matched by the ``collections`` argument will not be searched.
1121 storage_class : `str` or `StorageClass` or `None`
1122 A storage class to use when creating the returned entry. If given
1123 it must be compatible with the default storage class.
1124 dimension_records : `bool`, optional
1125 If `True` the ref will be expanded and contain dimension records.
1126 datastore_records : `bool`, optional
1127 If `True` the ref will contain associated datastore records.
1128 **kwargs
1129 Additional keyword arguments passed to
1130 `DataCoordinate.standardize` to convert ``dataId`` to a true
1131 `DataCoordinate` or augment an existing one. This can also include
1132 dimension record metadata that can be used to derive a primary
1133 dimension value.
1135 Returns
1136 -------
1137 ref : `DatasetRef`
1138 A reference to the dataset, or `None` if no matching Dataset
1139 was found.
1141 Raises
1142 ------
1143 lsst.daf.butler.NoDefaultCollectionError
1144 Raised if ``collections`` is `None` and
1145 ``self.collections`` is `None`.
1146 LookupError
1147 Raised if one or more data ID keys are missing.
1148 lsst.daf.butler.MissingDatasetTypeError
1149 Raised if the dataset type does not exist.
1150 lsst.daf.butler.MissingCollectionError
1151 Raised if any of ``collections`` does not exist in the registry.
1153 Notes
1154 -----
1155 This method simply returns `None` and does not raise an exception even
1156 when the set of collections searched is intrinsically incompatible with
1157 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
1158 only `~CollectionType.CALIBRATION` collections are being searched.
1159 This may make it harder to debug some lookup failures, but the behavior
1160 is intentional; we consider it more important that failed searches are
1161 reported consistently, regardless of the reason, and that adding
1162 additional collections that do not contain a match to the search path
1163 never changes the behavior.
1165 This method handles component dataset types automatically, though most
1166 other query operations do not.
1167 """
1168 raise NotImplementedError()
1170 @abstractmethod
1171 def retrieve_artifacts_zip(
1172 self,
1173 refs: Iterable[DatasetRef],
1174 destination: ResourcePathExpression,
1175 overwrite: bool = True,
1176 ) -> ResourcePath:
1177 """Retrieve artifacts from a Butler and place in ZIP file.
1179 Parameters
1180 ----------
1181 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
1182 The datasets to be included in the zip file.
1183 destination : `lsst.resources.ResourcePathExpression`
1184 Directory to write the new ZIP file. This directory will
1185 also be used as a staging area for the datasets being downloaded
1186 from the datastore.
1187 overwrite : `bool`, optional
1188 If `False` the output Zip will not be written if a file of the
1189 same name is already present in ``destination``.
1191 Returns
1192 -------
1193 zip_file : `lsst.resources.ResourcePath`
1194 The path to the new ZIP file.
1196 Raises
1197 ------
1198 ValueError
1199 Raised if there are no refs to retrieve.
1200 """
1201 raise NotImplementedError()
1203 @abstractmethod
1204 def retrieveArtifacts(
1205 self,
1206 refs: Iterable[DatasetRef],
1207 destination: ResourcePathExpression,
1208 transfer: str = "auto",
1209 preserve_path: bool = True,
1210 overwrite: bool = False,
1211 ) -> list[ResourcePath]:
1212 """Retrieve the artifacts associated with the supplied refs.
1214 Parameters
1215 ----------
1216 refs : iterable of `DatasetRef`
1217 The datasets for which artifacts are to be retrieved.
1218 A single ref can result in multiple artifacts. The refs must
1219 be resolved.
1220 destination : `lsst.resources.ResourcePath` or `str`
1221 Location to write the artifacts.
1222 transfer : `str`, optional
1223 Method to use to transfer the artifacts. Must be one of the options
1224 supported by `~lsst.resources.ResourcePath.transfer_from()`.
1225 "move" is not allowed.
1226 preserve_path : `bool`, optional
1227 If `True` the full path of the artifact within the datastore
1228 is preserved. If `False` the final file component of the path
1229 is used.
1230 overwrite : `bool`, optional
1231 If `True` allow transfers to overwrite existing files at the
1232 destination.
1234 Returns
1235 -------
1236 targets : `list` of `lsst.resources.ResourcePath`
1237 URIs of file artifacts in destination location. Order is not
1238 preserved.
1240 Notes
1241 -----
1242 For non-file datastores the artifacts written to the destination
1243 may not match the representation inside the datastore. For example
1244 a hierarchical data structure in a NoSQL database may well be stored
1245 as a JSON file.
1246 """
1247 raise NotImplementedError()
1249 @abstractmethod
1250 def exists(
1251 self,
1252 dataset_ref_or_type: DatasetRef | DatasetType | str,
1253 /,
1254 data_id: DataId | None = None,
1255 *,
1256 full_check: bool = True,
1257 collections: Any = None,
1258 **kwargs: Any,
1259 ) -> DatasetExistence:
1260 """Indicate whether a dataset is known to Butler registry and
1261 datastore.
1263 Parameters
1264 ----------
1265 dataset_ref_or_type : `DatasetRef`, `DatasetType`, or `str`
1266 When `DatasetRef` the `dataId` should be `None`.
1267 Otherwise the `DatasetType` or name thereof.
1268 data_id : `dict` or `DataCoordinate`
1269 A `dict` of `Dimension` link name, value pairs that label the
1270 `DatasetRef` within a Collection. When `None`, a `DatasetRef`
1271 should be provided as the first argument.
1272 full_check : `bool`, optional
1273 If `True`, a check will be made for the actual existence of a
1274 dataset artifact. This will involve additional overhead due to
1275 the need to query an external system. If `False`, this check will
1276 be omitted, and the registry and datastore will solely be asked
1277 if they know about the dataset but no direct check for the
1278 artifact will be performed.
1279 collections : Any, optional
1280 Collections to be searched, overriding ``self.collections``.
1281 Can be any of the types supported by the ``collections`` argument
1282 to butler construction.
1283 **kwargs
1284 Additional keyword arguments used to augment or construct a
1285 `DataCoordinate`. See `DataCoordinate.standardize`
1286 parameters.
1288 Returns
1289 -------
1290 existence : `DatasetExistence`
1291 Object indicating whether the dataset is known to registry and
1292 datastore. Evaluates to `True` if the dataset is present and known
1293 to both.
1294 """
1295 raise NotImplementedError()
1297 @abstractmethod
1298 def _exists_many(
1299 self,
1300 refs: Iterable[DatasetRef],
1301 /,
1302 *,
1303 full_check: bool = True,
1304 ) -> dict[DatasetRef, DatasetExistence]:
1305 """Indicate whether multiple datasets are known to Butler registry and
1306 datastore.
1308 This is an experimental API that may change at any moment.
1310 Parameters
1311 ----------
1312 refs : iterable of `DatasetRef`
1313 The datasets to be checked.
1314 full_check : `bool`, optional
1315 If `True`, a check will be made for the actual existence of each
1316 dataset artifact. This will involve additional overhead due to
1317 the need to query an external system. If `False`, this check will
1318 be omitted, and the registry and datastore will solely be asked
1319 if they know about the dataset(s) but no direct check for the
1320 artifact(s) will be performed.
1322 Returns
1323 -------
1324 existence : dict of [`DatasetRef`, `DatasetExistence`]
1325 Mapping from the given dataset refs to an enum indicating the
1326 status of the dataset in registry and datastore.
1327 Each value evaluates to `True` if the dataset is present and known
1328 to both.
1329 """
1330 raise NotImplementedError()
1332 @abstractmethod
1333 def removeRuns(
1334 self,
1335 names: Iterable[str],
1336 unstore: bool | type[_DeprecatedDefault] = _DeprecatedDefault,
1337 *,
1338 unlink_from_chains: bool = False,
1339 ) -> None:
1340 """Remove one or more `~CollectionType.RUN` collections and the
1341 datasets within them.
1343 Parameters
1344 ----------
1345 names : `~collections.abc.Iterable` [ `str` ]
1346 The names of the collections to remove.
1347 unstore : `bool`, optional
1348 If `True` (default), delete datasets from all datastores in which
1349 they are present, and attempt to rollback the registry deletions if
1350 datastore deletions fail (which may not always be possible). If
1351 `False`, datastore records for these datasets are still removed,
1352 but any artifacts (e.g. files) will not be. This parameter is now
1353 deprecated and no longer has any effect. Files are always deleted
1354 from datastores unless they were ingested using full URIs.
1355 unlink_from_chains : `bool`, optional
1356 If `True` remove the RUN collection from any chains prior to
1357 removing the RUN. If `False` the removal will fail if any chains
1358 still refer to the RUN.
1360 Raises
1361 ------
1362 TypeError
1363 Raised if one or more collections are not of type
1364 `~CollectionType.RUN`.
1365 """
1366 raise NotImplementedError()
1368 @abstractmethod
1369 def ingest(
1370 self,
1371 *datasets: FileDataset,
1372 transfer: str | None = "auto",
1373 record_validation_info: bool = True,
1374 skip_existing: bool = False,
1375 ) -> None:
1376 """Store and register one or more datasets that already exist on disk.
1378 Parameters
1379 ----------
1380 *datasets : `FileDataset`
1381 Each positional argument is a struct containing information about
1382 a file to be ingested, including its URI (either absolute or
1383 relative to the datastore root, if applicable), a resolved
1384 `DatasetRef`, and optionally a formatter class or its
1385 fully-qualified string name. If a formatter is not provided, the
1386 formatter that would be used for `put` is assumed. On successful
1387 ingest all `FileDataset.formatter` attributes will be set to the
1388 formatter class used. `FileDataset.path` attributes may be modified
1389 to put paths in whatever the datastore considers a standardized
1390 form.
1391 transfer : `str`, optional
1392 If not `None`, must be one of 'auto', 'move', 'copy', 'direct',
1393 'split', 'hardlink', 'relsymlink' or 'symlink', indicating how to
1394 transfer the file.
1395 record_validation_info : `bool`, optional
1396 If `True`, the default, the datastore can record validation
1397 information associated with the file. If `False` the datastore
1398 will not attempt to track any information such as checksums
1399 or file sizes. This can be useful if such information is tracked
1400 in an external system or if the file is to be compressed in place.
1401 It is up to the datastore whether this parameter is relevant.
1402 skip_existing : `bool`, optional
1403 If `True`, a dataset will not be ingested if a dataset with the
1404 same dataset ID already exists in the datastore.
1405 If `False` (the default), a `ConflictingDefinitionError` will be
1406 raised if any datasets with the same dataset ID already exist
1407 in the datastore.
1409 Returns
1410 -------
1411 None
1413 Raises
1414 ------
1415 TypeError
1416 Raised if the butler is read-only or if no run was provided.
1417 NotImplementedError
1418 Raised if the `Datastore` does not support the given transfer mode.
1419 DatasetTypeNotSupportedError
1420 Raised if one or more files to be ingested have a dataset type that
1421 is not supported by the `Datastore`..
1422 FileNotFoundError
1423 Raised if one of the given files does not exist.
1424 FileExistsError
1425 Raised if transfer is not `None` but the (internal) location the
1426 file would be moved to is already occupied.
1427 ConflictingDefinitionError
1428 Raised if a dataset already exists in the repository and
1429 ``skip_existing`` is `False`.
1431 Notes
1432 -----
1433 This operation is not fully exception safe: if a database operation
1434 fails, the given `FileDataset` instances may be only partially updated.
1436 It is atomic in terms of database operations (they will either all
1437 succeed or all fail) providing the database engine implements
1438 transactions correctly. It will attempt to be atomic in terms of
1439 filesystem operations as well, but this cannot be implemented
1440 rigorously for most datastores.
1441 """
1442 raise NotImplementedError()
1444 @abstractmethod
1445 def ingest_zip(
1446 self,
1447 zip_file: ResourcePathExpression,
1448 transfer: str = "auto",
1449 *,
1450 transfer_dimensions: bool = False,
1451 dry_run: bool = False,
1452 skip_existing: bool = False,
1453 ) -> None:
1454 """Ingest a Zip file into this butler.
1456 The Zip file must have been created by `retrieve_artifacts_zip`.
1458 Parameters
1459 ----------
1460 zip_file : `lsst.resources.ResourcePathExpression`
1461 Path to the Zip file.
1462 transfer : `str`, optional
1463 Method to use to transfer the Zip into the datastore.
1464 transfer_dimensions : `bool`, optional
1465 If `True`, dimension record data associated with the new datasets
1466 will be transferred from the Zip file, if present.
1467 dry_run : `bool`, optional
1468 If `True` the ingest will be processed without any modifications
1469 made to the target butler and as if the target butler did not
1470 have any of the datasets.
1471 skip_existing : `bool`, optional
1472 If `True`, a zip will not be ingested if the dataset entries listed
1473 in the index with the same dataset ID already exists in the butler.
1474 If `False` (the default), a `ConflictingDefinitionError` will be
1475 raised if any datasets with the same dataset ID already exist
1476 in the repository. If, somehow, some datasets are known to the
1477 butler and some are not, this is currently treated as an error
1478 rather than attempting to do a partial ingest.
1480 Notes
1481 -----
1482 Run collections and dataset types are created as needed.
1483 """
1484 raise NotImplementedError()
1486 @abstractmethod
1487 def export(
1488 self,
1489 *,
1490 directory: str | None = None,
1491 filename: str | None = None,
1492 format: str | None = None,
1493 transfer: str | None = None,
1494 ) -> AbstractContextManager[RepoExportContext]:
1495 """Export datasets from the repository represented by this `Butler`.
1497 This method is a context manager that returns a helper object
1498 (`RepoExportContext`) that is used to indicate what information from
1499 the repository should be exported.
1501 Parameters
1502 ----------
1503 directory : `str`, optional
1504 Directory dataset files should be written to if ``transfer`` is not
1505 `None`.
1506 filename : `str`, optional
1507 Name for the file that will include database information associated
1508 with the exported datasets. If this is not an absolute path and
1509 ``directory`` is not `None`, it will be written to ``directory``
1510 instead of the current working directory. Defaults to
1511 "export.{format}".
1512 format : `str`, optional
1513 File format for the database information file. If `None`, the
1514 extension of ``filename`` will be used.
1515 transfer : `str`, optional
1516 Transfer mode passed to `Datastore.export`.
1518 Raises
1519 ------
1520 TypeError
1521 Raised if the set of arguments passed is inconsistent.
1523 Examples
1524 --------
1525 Typically the `Registry.queryDataIds` and `Registry.queryDatasets`
1526 methods are used to provide the iterables over data IDs and/or datasets
1527 to be exported::
1529 with butler.export("exports.yaml") as export:
1530 # Export all flats, but none of the dimension element rows
1531 # (i.e. data ID information) associated with them.
1532 export.saveDatasets(
1533 butler.registry.queryDatasets("flat"), elements=()
1534 )
1535 # Export all datasets that start with "deepCoadd_" and all of
1536 # their associated data ID information.
1537 export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*"))
1538 """
1539 raise NotImplementedError()
1541 @abstractmethod
1542 def import_(
1543 self,
1544 *,
1545 directory: ResourcePathExpression | None = None,
1546 filename: ResourcePathExpression | TextIO | None = None,
1547 format: str | None = None,
1548 transfer: str | None = None,
1549 skip_dimensions: set | None = None,
1550 record_validation_info: bool = True,
1551 without_datastore: bool = False,
1552 ) -> None:
1553 """Import datasets into this repository that were exported from a
1554 different butler repository via `~lsst.daf.butler.Butler.export`.
1556 Parameters
1557 ----------
1558 directory : `~lsst.resources.ResourcePathExpression`, optional
1559 Directory containing dataset files to import from. If `None`,
1560 ``filename`` and all dataset file paths specified therein must
1561 be absolute.
1562 filename : `~lsst.resources.ResourcePathExpression` or `TextIO`
1563 A stream or name of file that contains database information
1564 associated with the exported datasets, typically generated by
1565 `~lsst.daf.butler.Butler.export`. If this a string (name) or
1566 `~lsst.resources.ResourcePath` and is not an absolute path,
1567 it will first be looked for relative to ``directory`` and if not
1568 found there it will be looked for in the current working
1569 directory. Defaults to "export.{format}".
1570 format : `str`, optional
1571 File format for ``filename``. If `None`, the extension of
1572 ``filename`` will be used.
1573 transfer : `str`, optional
1574 Transfer mode passed to `~lsst.daf.butler.Datastore.ingest`.
1575 skip_dimensions : `set`, optional
1576 Names of dimensions that should be skipped and not imported.
1577 record_validation_info : `bool`, optional
1578 If `True`, the default, the datastore can record validation
1579 information associated with the file. If `False` the datastore
1580 will not attempt to track any information such as checksums
1581 or file sizes. This can be useful if such information is tracked
1582 in an external system or if the file is to be compressed in place.
1583 It is up to the datastore whether this parameter is relevant.
1584 without_datastore : `bool`, optional
1585 If `True` only registry records will be imported and the datastore
1586 will be ignored.
1588 Raises
1589 ------
1590 TypeError
1591 Raised if the set of arguments passed is inconsistent, or if the
1592 butler is read-only.
1593 """
1594 raise NotImplementedError()
1596 @abstractmethod
1597 def transfer_dimension_records_from(
1598 self, source_butler: LimitedButler | Butler, source_refs: Iterable[DatasetRef | DataCoordinate]
1599 ) -> None:
1600 """Transfer dimension records to this Butler from another Butler.
1602 Parameters
1603 ----------
1604 source_butler : `LimitedButler` or `Butler`
1605 Butler from which the records are to be transferred. If data IDs
1606 in ``source_refs`` are not expanded then this has to be a full
1607 `Butler` whose registry will be used to expand data IDs. If the
1608 source refs contain coordinates that are used to populate other
1609 records then this will also need to be a full `Butler`.
1610 source_refs : iterable of `DatasetRef` or `DataCoordinate`
1611 Datasets or data IDs defined in the source butler whose dimension
1612 records should be transferred to this butler.
1613 """
1614 raise NotImplementedError()
1616 @abstractmethod
1617 def transfer_from(
1618 self,
1619 source_butler: LimitedButler,
1620 source_refs: Iterable[DatasetRef],
1621 transfer: str = "auto",
1622 skip_missing: bool = True,
1623 register_dataset_types: bool = False,
1624 transfer_dimensions: bool = False,
1625 dry_run: bool = False,
1626 ) -> Collection[DatasetRef]:
1627 """Transfer datasets to this Butler from a run in another Butler.
1629 Parameters
1630 ----------
1631 source_butler : `LimitedButler`
1632 Butler from which the datasets are to be transferred. If data IDs
1633 in ``source_refs`` are not expanded then this has to be a full
1634 `Butler` whose registry will be used to expand data IDs.
1635 source_refs : iterable of `DatasetRef`
1636 Datasets defined in the source butler that should be transferred to
1637 this butler. In most circumstances, ``transfer_from`` is faster if
1638 the dataset refs are expanded.
1639 transfer : `str`, optional
1640 Transfer mode passed to `~lsst.daf.butler.Datastore.transfer_from`.
1641 skip_missing : `bool`
1642 If `True`, datasets with no datastore artifact associated with
1643 them are not transferred. If `False` a registry entry will be
1644 created even if no datastore record is created (and so will
1645 look equivalent to the dataset being unstored).
1646 register_dataset_types : `bool`
1647 If `True` any missing dataset types are registered. Otherwise
1648 an exception is raised.
1649 transfer_dimensions : `bool`, optional
1650 If `True`, dimension record data associated with the new datasets
1651 will be transferred.
1652 dry_run : `bool`, optional
1653 If `True` the transfer will be processed without any modifications
1654 made to the target butler and as if the target butler did not
1655 have any of the datasets.
1657 Returns
1658 -------
1659 refs : `list` of `DatasetRef`
1660 The refs added to this Butler.
1662 Notes
1663 -----
1664 The datastore artifact has to exist for a transfer
1665 to be made but non-existence is not an error.
1667 Datasets that already exist in this run will be skipped.
1669 The datasets are imported as part of a transaction, although
1670 dataset types are registered before the transaction is started.
1671 This means that it is possible for a dataset type to be registered
1672 even though transfer has failed.
1673 """
1674 raise NotImplementedError()
1676 @abstractmethod
1677 def validateConfiguration(
1678 self,
1679 logFailures: bool = False,
1680 datasetTypeNames: Iterable[str] | None = None,
1681 ignore: Iterable[str] | None = None,
1682 ) -> None:
1683 """Validate butler configuration.
1685 Checks that each `DatasetType` can be stored in the `Datastore`.
1687 Parameters
1688 ----------
1689 logFailures : `bool`, optional
1690 If `True`, output a log message for every validation error
1691 detected.
1692 datasetTypeNames : iterable of `str`, optional
1693 The `DatasetType` names that should be checked. This allows
1694 only a subset to be selected.
1695 ignore : iterable of `str`, optional
1696 Names of DatasetTypes to skip over. This can be used to skip
1697 known problems. If a named `DatasetType` corresponds to a
1698 composite, all components of that `DatasetType` will also be
1699 ignored.
1701 Raises
1702 ------
1703 ButlerValidationError
1704 Raised if there is some inconsistency with how this Butler
1705 is configured.
1706 """
1707 raise NotImplementedError()
1709 @property
1710 @abstractmethod
1711 def collection_chains(self) -> ButlerCollections:
1712 """Object with methods for modifying collection chains
1713 (`~lsst.daf.butler.ButlerCollections`).
1715 Deprecated. Replaced with ``collections`` property.
1716 """
1717 raise NotImplementedError()
1719 @property
1720 @abstractmethod
1721 def collections(self) -> ButlerCollections:
1722 """Object with methods for modifying and querying collections
1723 (`~lsst.daf.butler.ButlerCollections`).
1725 Use of this object is preferred over `registry` wherever possible.
1726 """
1727 raise NotImplementedError()
1729 @property
1730 @abstractmethod
1731 def run(self) -> str | None:
1732 """Name of the run this butler writes outputs to by default (`str` or
1733 `None`).
1734 """
1735 raise NotImplementedError()
1737 @property
1738 @abstractmethod
1739 def registry(self) -> Registry:
1740 """The object that manages dataset metadata and relationships
1741 (`Registry`).
1743 Many operations that don't involve reading or writing butler datasets
1744 are accessible only via `Registry` methods. Eventually these methods
1745 will be replaced by equivalent `Butler` methods.
1746 """
1747 raise NotImplementedError()
1749 @abstractmethod
1750 def query(self) -> AbstractContextManager[Query]:
1751 """Context manager returning a `Query` object used for construction
1752 and execution of complex queries.
1753 """
1754 raise NotImplementedError()
1756 def query_data_ids(
1757 self,
1758 dimensions: DimensionGroup | Iterable[str] | str,
1759 *,
1760 data_id: DataId | None = None,
1761 where: str = "",
1762 bind: Mapping[str, Any] | None = None,
1763 with_dimension_records: bool = False,
1764 order_by: Iterable[str] | str | None = None,
1765 limit: int | None = -20_000,
1766 explain: bool = True,
1767 **kwargs: Any,
1768 ) -> list[DataCoordinate]:
1769 """Query for data IDs matching user-provided criteria.
1771 Parameters
1772 ----------
1773 dimensions : `DimensionGroup`, `str`, or \
1774 `~collections.abc.Iterable` [`str`]
1775 The dimensions of the data IDs to yield, as either `DimensionGroup`
1776 instances or `str`. Will be automatically expanded to a complete
1777 `DimensionGroup`.
1778 data_id : `dict` or `DataCoordinate`, optional
1779 A data ID whose key-value pairs are used as equality constraints
1780 in the query.
1781 where : `str`, optional
1782 A string expression similar to a SQL WHERE clause. May involve
1783 any column of a dimension table or (as a shortcut for the primary
1784 key column of a dimension table) dimension name. See
1785 :ref:`daf_butler_dimension_expressions` for more information.
1786 bind : `~collections.abc.Mapping`, optional
1787 Mapping containing literal values that should be injected into the
1788 ``where`` expression, keyed by the identifiers they replace.
1789 Values of collection type can be expanded in some cases; see
1790 :ref:`daf_butler_dimension_expressions_identifiers` for more
1791 information.
1792 with_dimension_records : `bool`, optional
1793 If `True` (default is `False`) then returned data IDs will have
1794 dimension records.
1795 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1796 Names of the columns/dimensions to use for ordering returned data
1797 IDs. Column name can be prefixed with minus (``-``) to use
1798 descending ordering.
1799 limit : `int` or `None`, optional
1800 Upper limit on the number of returned records. `None` can be used
1801 if no limit is wanted. A limit of ``0`` means that the query will
1802 be executed and validated but no results will be returned. In this
1803 case there will be no exception even if ``explain`` is `True`.
1804 If a negative value is given a warning will be issued if the number
1805 of results is capped by that limit.
1806 explain : `bool`, optional
1807 If `True` (default) then `EmptyQueryResultError` exception is
1808 raised when resulting list is empty. The exception contains
1809 non-empty list of strings explaining possible causes for empty
1810 result.
1811 **kwargs
1812 Additional keyword arguments are forwarded to
1813 `DataCoordinate.standardize` when processing the ``data_id``
1814 argument (and may be used to provide a constraining data ID even
1815 when the ``data_id`` argument is `None`).
1817 Returns
1818 -------
1819 dataIds : `list` [`DataCoordinate`]
1820 Data IDs matching the given query parameters. These are always
1821 guaranteed to identify all dimensions (`DataCoordinate.hasFull`
1822 returns `True`).
1824 Raises
1825 ------
1826 lsst.daf.butler.registry.DataIdError
1827 Raised when ``data_id`` or keyword arguments specify unknown
1828 dimensions or values, or when they contain inconsistent values.
1829 lsst.daf.butler.registry.UserExpressionError
1830 Raised when ``where`` expression is invalid.
1831 lsst.daf.butler.EmptyQueryResultError
1832 Raised when query generates empty result and ``explain`` is set to
1833 `True`.
1834 TypeError
1835 Raised when the arguments are incompatible.
1836 """
1837 if data_id is None:
1838 data_id = DataCoordinate.make_empty(self.dimensions)
1839 if order_by is None:
1840 order_by = []
1841 query_limit = limit
1842 warn_limit = False
1843 if limit is not None and limit < 0:
1844 query_limit = abs(limit) + 1
1845 warn_limit = True
1846 with self.query() as query:
1847 result = (
1848 query.data_ids(dimensions)
1849 .where(data_id, where, bind=bind, **kwargs)
1850 .order_by(*ensure_iterable(order_by))
1851 .limit(query_limit)
1852 )
1853 if with_dimension_records:
1854 result = result.with_dimension_records()
1855 data_ids = list(result)
1856 if warn_limit and len(data_ids) == query_limit:
1857 # We asked for one too many so must remove that from the list.
1858 data_ids.pop(-1)
1859 assert limit is not None # For mypy.
1860 _LOG.warning("More data IDs are available than the requested limit of %d.", abs(limit))
1861 if explain and (limit is None or limit != 0) and not data_ids:
1862 raise EmptyQueryResultError(list(result.explain_no_results()))
1863 return data_ids
1865 def query_datasets(
1866 self,
1867 dataset_type: str | DatasetType,
1868 collections: str | Iterable[str] | None = None,
1869 *,
1870 find_first: bool = True,
1871 data_id: DataId | None = None,
1872 where: str = "",
1873 bind: Mapping[str, Any] | None = None,
1874 with_dimension_records: bool = False,
1875 order_by: Iterable[str] | str | None = None,
1876 limit: int | None = -20_000,
1877 explain: bool = True,
1878 **kwargs: Any,
1879 ) -> list[DatasetRef]:
1880 """Query for dataset references matching user-provided criteria.
1882 Parameters
1883 ----------
1884 dataset_type : `str` or `DatasetType`
1885 Dataset type object or name to search for.
1886 collections : collection expression, optional
1887 A collection name or iterable of collection names to search. If not
1888 provided, the default collections are used. Can be a wildcard if
1889 ``find_first`` is `False` (if find first is requested the order
1890 of collections matters and wildcards make the order indeterminate).
1891 See :ref:`daf_butler_collection_expressions` for more information.
1892 find_first : `bool`, optional
1893 If `True` (default), for each result data ID, only yield one
1894 `DatasetRef` of each `DatasetType`, from the first collection in
1895 which a dataset of that dataset type appears (according to the
1896 order of ``collections`` passed in). If `True`, ``collections``
1897 must not contain wildcards.
1898 data_id : `dict` or `DataCoordinate`, optional
1899 A data ID whose key-value pairs are used as equality constraints in
1900 the query.
1901 where : `str`, optional
1902 A string expression similar to a SQL WHERE clause. May involve any
1903 column of a dimension table or (as a shortcut for the primary key
1904 column of a dimension table) dimension name. See
1905 :ref:`daf_butler_dimension_expressions` for more information.
1906 bind : `~collections.abc.Mapping`, optional
1907 Mapping containing literal values that should be injected into the
1908 ``where`` expression, keyed by the identifiers they replace. Values
1909 of collection type can be expanded in some cases; see
1910 :ref:`daf_butler_dimension_expressions_identifiers` for more
1911 information.
1912 with_dimension_records : `bool`, optional
1913 If `True` (default is `False`) then returned data IDs will have
1914 dimension records.
1915 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
1916 Names of the columns/dimensions to use for ordering returned data
1917 IDs. Column name can be prefixed with minus (``-``) to use
1918 descending ordering.
1919 limit : `int` or `None`, optional
1920 Upper limit on the number of returned records. `None` can be used
1921 if no limit is wanted. A limit of ``0`` means that the query will
1922 be executed and validated but no results will be returned. In this
1923 case there will be no exception even if ``explain`` is `True`.
1924 If a negative value is given a warning will be issued if the number
1925 of results is capped by that limit.
1926 explain : `bool`, optional
1927 If `True` (default) then `EmptyQueryResultError` exception is
1928 raised when resulting list is empty. The exception contains
1929 non-empty list of strings explaining possible causes for empty
1930 result.
1931 **kwargs
1932 Additional keyword arguments are forwarded to
1933 `DataCoordinate.standardize` when processing the ``data_id``
1934 argument (and may be used to provide a constraining data ID even
1935 when the ``data_id`` argument is `None`).
1937 Returns
1938 -------
1939 refs : `.queries.DatasetRefQueryResults`
1940 Dataset references matching the given query criteria. Nested data
1941 IDs are guaranteed to include values for all implied dimensions
1942 (i.e. `DataCoordinate.hasFull` will return `True`).
1944 Raises
1945 ------
1946 lsst.daf.butler.registry.DatasetTypeExpressionError
1947 Raised when ``dataset_type`` expression is invalid.
1948 lsst.daf.butler.registry.DataIdError
1949 Raised when ``data_id`` or keyword arguments specify unknown
1950 dimensions or values, or when they contain inconsistent values.
1951 lsst.daf.butler.registry.UserExpressionError
1952 Raised when ``where`` expression is invalid.
1953 lsst.daf.butler.EmptyQueryResultError
1954 Raised when query generates empty result and ``explain`` is set to
1955 `True`.
1956 TypeError
1957 Raised when the arguments are incompatible, such as when a
1958 collection wildcard is passed when ``find_first`` is `True`, or
1959 when ``collections`` is `None` and default butler collections are
1960 not defined.
1961 """
1962 if data_id is None:
1963 data_id = DataCoordinate.make_empty(self.dimensions)
1964 if order_by is None:
1965 order_by = []
1966 if collections and has_globs(collections):
1967 # Wild cards need to be expanded but can only be allowed if
1968 # find_first=False because expanding wildcards does not return
1969 # a guaranteed ordering. Querying collection registry to expand
1970 # collections when we do not have wildcards is expensive so only
1971 # do it if we need it.
1972 if find_first:
1973 raise InvalidQueryError(
1974 f"Can not use wildcards in collections when find_first=True (given {collections})"
1975 )
1976 collections = self.collections.query(collections)
1977 query_limit = limit
1978 warn_limit = False
1979 if limit is not None and limit < 0:
1980 query_limit = abs(limit) + 1
1981 warn_limit = True
1982 with self.query() as query:
1983 result = (
1984 query.datasets(dataset_type, collections=collections, find_first=find_first)
1985 .where(data_id, where, bind=bind, **kwargs)
1986 .order_by(*ensure_iterable(order_by))
1987 .limit(query_limit)
1988 )
1989 if with_dimension_records:
1990 result = result.with_dimension_records()
1991 refs = list(result)
1992 if warn_limit and len(refs) == query_limit:
1993 # We asked for one too many so must remove that from the list.
1994 refs.pop(-1)
1995 assert limit is not None # For mypy.
1996 _LOG.warning("More datasets are available than the requested limit of %d.", abs(limit))
1997 if explain and (limit is None or limit != 0) and not refs:
1998 raise EmptyQueryResultError(list(result.explain_no_results()))
1999 return refs
2001 def query_dimension_records(
2002 self,
2003 element: str,
2004 *,
2005 data_id: DataId | None = None,
2006 where: str = "",
2007 bind: Mapping[str, Any] | None = None,
2008 order_by: Iterable[str] | str | None = None,
2009 limit: int | None = -20_000,
2010 explain: bool = True,
2011 **kwargs: Any,
2012 ) -> list[DimensionRecord]:
2013 """Query for dimension information matching user-provided criteria.
2015 Parameters
2016 ----------
2017 element : `str`
2018 The name of a dimension element to obtain records for.
2019 data_id : `dict` or `DataCoordinate`, optional
2020 A data ID whose key-value pairs are used as equality constraints
2021 in the query.
2022 where : `str`, optional
2023 A string expression similar to a SQL WHERE clause. See
2024 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
2025 information.
2026 bind : `~collections.abc.Mapping`, optional
2027 Mapping containing literal values that should be injected into the
2028 ``where`` expression, keyed by the identifiers they replace.
2029 Values of collection type can be expanded in some cases; see
2030 :ref:`daf_butler_dimension_expressions_identifiers` for more
2031 information.
2032 order_by : `~collections.abc.Iterable` [`str`] or `str`, optional
2033 Names of the columns/dimensions to use for ordering returned data
2034 IDs. Column name can be prefixed with minus (``-``) to use
2035 descending ordering.
2036 limit : `int` or `None`, optional
2037 Upper limit on the number of returned records. `None` can be used
2038 if no limit is wanted. A limit of ``0`` means that the query will
2039 be executed and validated but no results will be returned. In this
2040 case there will be no exception even if ``explain`` is `True`.
2041 If a negative value is given a warning will be issued if the number
2042 of results is capped by that limit.
2043 explain : `bool`, optional
2044 If `True` (default) then `EmptyQueryResultError` exception is
2045 raised when resulting list is empty. The exception contains
2046 non-empty list of strings explaining possible causes for empty
2047 result.
2048 **kwargs
2049 Additional keyword arguments are forwarded to
2050 `DataCoordinate.standardize` when processing the ``data_id``
2051 argument (and may be used to provide a constraining data ID even
2052 when the ``data_id`` argument is `None`).
2054 Returns
2055 -------
2056 records : `list` [`DimensionRecord`]
2057 Dimension records matching the given query parameters.
2059 Raises
2060 ------
2061 lsst.daf.butler.registry.DataIdError
2062 Raised when ``data_id`` or keyword arguments specify unknown
2063 dimensions or values, or when they contain inconsistent values.
2064 lsst.daf.butler.registry.UserExpressionError
2065 Raised when ``where`` expression is invalid.
2066 lsst.daf.butler.EmptyQueryResultError
2067 Raised when query generates empty result and ``explain`` is set to
2068 `True`.
2069 TypeError
2070 Raised when the arguments are incompatible, such as when a
2071 collection wildcard is passed when ``find_first`` is `True`, or
2072 when ``collections`` is `None` and default butler collections are
2073 not defined.
2074 """
2075 if data_id is None:
2076 data_id = DataCoordinate.make_empty(self.dimensions)
2077 if order_by is None:
2078 order_by = []
2079 query_limit = limit
2080 warn_limit = False
2081 if limit is not None and limit < 0:
2082 query_limit = abs(limit) + 1
2083 warn_limit = True
2084 with self.query() as query:
2085 result = (
2086 query.dimension_records(element)
2087 .where(data_id, where, bind=bind, **kwargs)
2088 .order_by(*ensure_iterable(order_by))
2089 .limit(query_limit)
2090 )
2091 dimension_records = list(result)
2092 if warn_limit and len(dimension_records) == query_limit:
2093 # We asked for one too many so must remove that from the list.
2094 dimension_records.pop(-1)
2095 assert limit is not None # For mypy.
2096 _LOG.warning(
2097 "More dimension records are available than the requested limit of %d.", abs(limit)
2098 )
2099 if explain and (limit is None or limit != 0) and not dimension_records:
2100 raise EmptyQueryResultError(list(result.explain_no_results()))
2101 return dimension_records
2103 def query_all_datasets(
2104 self,
2105 collections: str | Iterable[str] | None = None,
2106 *,
2107 name: str | Iterable[str] = "*",
2108 find_first: bool = True,
2109 data_id: DataId | None = None,
2110 where: str = "",
2111 bind: Mapping[str, Any] | None = None,
2112 limit: int | None = -20_000,
2113 **kwargs: Any,
2114 ) -> list[DatasetRef]:
2115 """Query for datasets of potentially multiple types.
2117 Parameters
2118 ----------
2119 collections : `str` or `~collections.abc.Iterable` [ `str` ], optional
2120 The collection or collections to search, in order. If not provided
2121 or `None`, the default collection search path for this butler is
2122 used.
2123 name : `str` or `~collections.abc.Iterable` [ `str` ], optional
2124 Names or name patterns (glob-style) that returned dataset type
2125 names must match. If an iterable, items are OR'd together. The
2126 default is to include all dataset types in the given collections.
2127 find_first : `bool`, optional
2128 If `True` (default), for each result data ID, only yield one
2129 `DatasetRef` of each `DatasetType`, from the first collection in
2130 which a dataset of that dataset type appears (according to the
2131 order of ``collections`` passed in).
2132 data_id : `dict` or `DataCoordinate`, optional
2133 A data ID whose key-value pairs are used as equality constraints in
2134 the query.
2135 where : `str`, optional
2136 A string expression similar to a SQL WHERE clause. May involve any
2137 column of a dimension table or (as a shortcut for the primary key
2138 column of a dimension table) dimension name. See
2139 :ref:`daf_butler_dimension_expressions` for more information.
2140 bind : `~collections.abc.Mapping`, optional
2141 Mapping containing literal values that should be injected into the
2142 ``where`` expression, keyed by the identifiers they replace. Values
2143 of collection type can be expanded in some cases; see
2144 :ref:`daf_butler_dimension_expressions_identifiers` for more
2145 information.
2146 limit : `int` or `None`, optional
2147 Upper limit on the number of returned records. `None` can be used
2148 if no limit is wanted. A limit of ``0`` means that the query will
2149 be executed and validated but no results will be returned.
2150 If a negative value is given a warning will be issued if the number
2151 of results is capped by that limit. If no limit is provided, by
2152 default a maximum of 20,000 records will be returned.
2153 **kwargs
2154 Additional keyword arguments are forwarded to
2155 `DataCoordinate.standardize` when processing the ``data_id``
2156 argument (and may be used to provide a constraining data ID even
2157 when the ``data_id`` argument is `None`).
2159 Raises
2160 ------
2161 MissingDatasetTypeError
2162 When no dataset types match ``name``, or an explicit (non-glob)
2163 dataset type in ``name`` does not exist.
2164 InvalidQueryError
2165 If the parameters to the query are inconsistent or malformed.
2166 MissingCollectionError
2167 If a given collection is not found.
2169 Returns
2170 -------
2171 refs : `list` [ `DatasetRef` ]
2172 Dataset references matching the given query criteria. Nested data
2173 IDs are guaranteed to include values for all implied dimensions
2174 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
2175 include dimension records (`DataCoordinate.hasRecords` will be
2176 `False`).
2177 """
2178 if collections is None:
2179 collections = list(self.collections.defaults)
2180 else:
2181 collections = list(ensure_iterable(collections))
2183 if bind is None:
2184 bind = {}
2185 if data_id is None:
2186 data_id = {}
2188 warn_limit = False
2189 if limit is not None and limit < 0:
2190 # Add one to the limit so we can detect if we have exceeded it.
2191 limit = abs(limit) + 1
2192 warn_limit = True
2194 args = QueryAllDatasetsParameters(
2195 collections=collections,
2196 name=list(ensure_iterable(name)),
2197 find_first=find_first,
2198 data_id=data_id,
2199 where=where,
2200 limit=limit,
2201 bind=bind,
2202 kwargs=kwargs,
2203 with_dimension_records=False,
2204 )
2205 with self._query_all_datasets_by_page(args) as pages:
2206 result = []
2207 for page in pages:
2208 result.extend(page)
2210 if warn_limit and limit is not None and len(result) >= limit:
2211 # Remove the extra dataset we added for the limit check.
2212 result.pop()
2213 _LOG.warning("More datasets are available than the requested limit of %d.", limit - 1)
2215 return result
2217 @abstractmethod
2218 def _query_all_datasets_by_page(
2219 self, args: QueryAllDatasetsParameters
2220 ) -> AbstractContextManager[Iterator[list[DatasetRef]]]:
2221 raise NotImplementedError()
2223 def clone(
2224 self,
2225 *,
2226 collections: CollectionArgType | None | EllipsisType = ...,
2227 run: str | None | EllipsisType = ...,
2228 inferDefaults: bool | EllipsisType = ...,
2229 dataId: dict[str, str] | EllipsisType = ...,
2230 metrics: ButlerMetrics | None = None,
2231 ) -> Butler:
2232 """Return a new Butler instance connected to the same repository
2233 as this one, optionally overriding ``collections``, ``run``,
2234 ``inferDefaults``, and default data ID.
2236 Parameters
2237 ----------
2238 collections : `~lsst.daf.butler.registry.CollectionArgType` or `None`,\
2239 optional
2240 Same as constructor. If omitted, uses value from original object.
2241 run : `str` or `None`, optional
2242 Same as constructor. If `None`, no default run is used. If
2243 omitted, copies value from original object.
2244 inferDefaults : `bool`, optional
2245 Same as constructor. If omitted, copies value from original
2246 object.
2247 dataId : `str`
2248 Same as ``kwargs`` passed to the constructor. If omitted, copies
2249 values from original object.
2250 metrics : `ButlerMetrics` or `None`, optional
2251 Metrics object to record butler statistics.
2252 """
2253 raise NotImplementedError()
2255 @abstractmethod
2256 def close(self) -> None:
2257 raise NotImplementedError()
2259 @abstractmethod
2260 def _expand_data_ids(self, data_ids: Iterable[DataCoordinate]) -> list[DataCoordinate]:
2261 raise NotImplementedError()