Coverage for python/lsst/daf/butler/script/transferDatasets.py: 37%
17 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-02 18:18 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-02 18:18 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("transferDatasets",)
24import logging
25from typing import Tuple
27from .._butler import Butler
28from ..registry.queries import DatasetQueryResults
30log = logging.getLogger(__name__)
33def transferDatasets(
34 source: str,
35 dest: str,
36 dataset_type: Tuple[str, ...],
37 collections: Tuple[str, ...],
38 where: str,
39 find_first: bool,
40 transfer: str,
41 register_dataset_types: bool,
42) -> int:
43 """Transfer datasets from run in source to dest.
45 Parameters
46 ----------
47 source : `str`
48 URI string of the source Butler repo.
49 dest : `str`
50 URI string of the destination Butler repo.
51 dataset_type : `tuple` of `str`
52 Dataset type names. An empty tuple implies all dataset types.
53 collections : `tuple` of `str`
54 Names of collection globs to match. An empty tuple implies all
55 collections.
56 where : `str`
57 Query modification string.
58 find_first : `bool`
59 Whether only the first match should be used.
60 transfer : `str`
61 Transfer mode to use when placing artifacts in the destination.
62 register_dataset_types : `bool`
63 Indicate whether missing dataset types should be registered.
64 """
65 source_butler = Butler(source, writeable=False)
66 dest_butler = Butler(dest, writeable=True)
68 dataset_type_expr = ... if not dataset_type else dataset_type
69 collections_expr = ... if not collections else collections
71 source_refs = source_butler.registry.queryDatasets(
72 datasetType=dataset_type_expr, collections=collections_expr, where=where, findFirst=find_first
73 )
75 # Might need expanded results if datastore records have to be derived.
76 # Not all registries return the same form for results.
77 if isinstance(source_refs, DatasetQueryResults):
78 source_refs = source_refs.expanded()
80 # Place results in a set to remove duplicates
81 source_refs_set = set(source_refs)
83 transferred = dest_butler.transfer_from(
84 source_butler, source_refs_set, transfer=transfer, register_dataset_types=register_dataset_types
85 )
86 return len(transferred)