Coverage for python/lsst/daf/butler/script/retrieveArtifacts.py: 53%
15 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-05 10:00 +0000
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-05 10:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("retrieveArtifacts",)
32import logging
33from types import EllipsisType
34from typing import TYPE_CHECKING
36from .._butler import Butler
38if TYPE_CHECKING:
39 from lsst.resources import ResourcePath
41log = logging.getLogger(__name__)
44def retrieveArtifacts(
45 repo: str,
46 destination: str,
47 dataset_type: tuple[str, ...],
48 collections: tuple[str, ...],
49 where: str,
50 find_first: bool,
51 transfer: str,
52 preserve_path: bool,
53 clobber: bool,
54) -> list[ResourcePath]:
55 """Parameters are those required for querying datasets plus a destination
56 URI.
58 Parameters
59 ----------
60 repo : `str`
61 URI string of the Butler repo to use.
62 destination : `str`
63 URI string of the directory to write the artifacts.
64 dataset_type : `tuple` of `str`
65 Dataset type names. An empty tuple implies all dataset types.
66 collections : `tuple` of `str`
67 Names of collection globs to match. An empty tuple implies all
68 collections.
69 where : `str`
70 Query modification string.
71 find_first : `bool`
72 Whether only the first match should be used.
73 transfer : `str`
74 Transfer mode to use when placing artifacts in the destination.
75 preserve_path : `bool`
76 If `True` the full datastore path will be retained within the
77 destination directory, else only the filename will be used.
78 clobber : `bool`
79 If `True` allow transfers to overwrite files at the destination.
81 Returns
82 -------
83 transferred : `list` of `lsst.resources.ResourcePath`
84 The destination URIs of every transferred artifact.
85 """
86 query_types = dataset_type or ...
87 query_collections: tuple[str, ...] | EllipsisType = collections or ...
89 butler = Butler.from_config(repo, writeable=False)
91 # Need to store in list so we can count the number to give some feedback
92 # to caller.
93 refs = list(
94 butler.registry.queryDatasets(
95 datasetType=query_types, collections=query_collections, where=where, findFirst=find_first
96 )
97 )
99 log.info("Number of datasets matching query: %d", len(refs))
101 transferred = butler.retrieveArtifacts(
102 refs, destination=destination, transfer=transfer, preserve_path=preserve_path, overwrite=clobber
103 )
104 return transferred