Coverage for python/lsst/daf/butler/script/retrieveArtifacts.py: 53%
15 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 09:30 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 09:30 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("retrieveArtifacts",)
26import logging
27from types import EllipsisType
28from typing import TYPE_CHECKING
30from .._butler import Butler
32if TYPE_CHECKING:
33 from lsst.resources import ResourcePath
35log = logging.getLogger(__name__)
38def retrieveArtifacts(
39 repo: str,
40 destination: str,
41 dataset_type: tuple[str, ...],
42 collections: tuple[str, ...],
43 where: str,
44 find_first: bool,
45 transfer: str,
46 preserve_path: bool,
47 clobber: bool,
48) -> list[ResourcePath]:
49 """Parameters are those required for querying datasets plus a destination
50 URI.
52 Parameters
53 ----------
54 repo : `str`
55 URI string of the Butler repo to use.
56 destination : `str`
57 URI string of the directory to write the artifacts.
58 dataset_type : `tuple` of `str`
59 Dataset type names. An empty tuple implies all dataset types.
60 collections : `tuple` of `str`
61 Names of collection globs to match. An empty tuple implies all
62 collections.
63 where : `str`
64 Query modification string.
65 find_first : `bool`
66 Whether only the first match should be used.
67 transfer : `str`
68 Transfer mode to use when placing artifacts in the destination.
69 preserve_path : `bool`
70 If `True` the full datastore path will be retained within the
71 destination directory, else only the filename will be used.
72 clobber : `bool`
73 If `True` allow transfers to overwrite files at the destination.
75 Returns
76 -------
77 transferred : `list` of `lsst.resources.ResourcePath`
78 The destination URIs of every transferred artifact.
79 """
80 query_types = dataset_type if dataset_type else ...
81 query_collections: tuple[str, ...] | EllipsisType = collections if collections else ...
83 butler = Butler(repo, writeable=False)
85 # Need to store in list so we can count the number to give some feedback
86 # to caller.
87 refs = list(
88 butler.registry.queryDatasets(
89 datasetType=query_types, collections=query_collections, where=where, findFirst=find_first
90 )
91 )
93 log.info("Number of datasets matching query: %d", len(refs))
95 transferred = butler.retrieveArtifacts(
96 refs, destination=destination, transfer=transfer, preserve_path=preserve_path, overwrite=clobber
97 )
98 return transferred