Coverage for python/lsst/daf/butler/script/retrieveArtifacts.py: 53%
15 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-18 09:13 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-18 09:13 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("retrieveArtifacts",)
26import logging
27from typing import TYPE_CHECKING
29from lsst.utils.ellipsis import Ellipsis, EllipsisType
31from .._butler import Butler
33if TYPE_CHECKING:
34 from lsst.resources import ResourcePath
36log = logging.getLogger(__name__)
39def retrieveArtifacts(
40 repo: str,
41 destination: str,
42 dataset_type: tuple[str, ...],
43 collections: tuple[str, ...],
44 where: str,
45 find_first: bool,
46 transfer: str,
47 preserve_path: bool,
48 clobber: bool,
49) -> list[ResourcePath]:
50 """Parameters are those required for querying datasets plus a destination
51 URI.
53 Parameters
54 ----------
55 repo : `str`
56 URI string of the Butler repo to use.
57 destination : `str`
58 URI string of the directory to write the artifacts.
59 dataset_type : `tuple` of `str`
60 Dataset type names. An empty tuple implies all dataset types.
61 collections : `tuple` of `str`
62 Names of collection globs to match. An empty tuple implies all
63 collections.
64 where : `str`
65 Query modification string.
66 find_first : `bool`
67 Whether only the first match should be used.
68 transfer : `str`
69 Transfer mode to use when placing artifacts in the destination.
70 preserve_path : `bool`
71 If `True` the full datastore path will be retained within the
72 destination directory, else only the filename will be used.
73 clobber : `bool`
74 If `True` allow transfers to overwrite files at the destination.
76 Returns
77 -------
78 transferred : `list` of `lsst.resources.ResourcePath`
79 The destination URIs of every transferred artifact.
80 """
81 query_types = dataset_type if dataset_type else ...
82 query_collections: tuple[str, ...] | EllipsisType = collections if collections else Ellipsis
84 butler = Butler(repo, writeable=False)
86 # Need to store in list so we can count the number to give some feedback
87 # to caller.
88 refs = list(
89 butler.registry.queryDatasets(
90 datasetType=query_types, collections=query_collections, where=where, findFirst=find_first
91 )
92 )
94 log.info("Number of datasets matching query: %d", len(refs))
96 transferred = butler.retrieveArtifacts(
97 refs, destination=destination, transfer=transfer, preserve_path=preserve_path, overwrite=clobber
98 )
99 return transferred