Coverage for python/lsst/daf/butler/script/transferDatasets.py: 37%
17 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:54 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:54 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("transferDatasets",)
24import logging
25from typing import Tuple
27from .. import Butler
28from ..registry.queries import DatasetQueryResults
30log = logging.getLogger(__name__)
33def transferDatasets(source: str, dest: str, dataset_type: Tuple[str, ...], collections: Tuple[str, ...],
34 where: str, find_first: bool,
35 transfer: str, register_dataset_types: bool) -> int:
36 """Transfer datasets from run in source to dest.
38 Parameters
39 ----------
40 source : `str`
41 URI string of the source Butler repo.
42 dest : `str`
43 URI string of the destination Butler repo.
44 dataset_type : `tuple` of `str`
45 Dataset type names. An empty tuple implies all dataset types.
46 collections : `tuple` of `str`
47 Names of collection globs to match. An empty tuple implies all
48 collections.
49 where : `str`
50 Query modification string.
51 find_first : `bool`
52 Whether only the first match should be used.
53 transfer : `str`
54 Transfer mode to use when placing artifacts in the destination.
55 register_dataset_types : `bool`
56 Indicate whether missing dataset types should be registered.
57 """
58 source_butler = Butler(source, writeable=False)
59 dest_butler = Butler(dest, writeable=True)
61 dataset_type_expr = ... if not dataset_type else dataset_type
62 collections_expr = ... if not collections else collections
64 source_refs = source_butler.registry.queryDatasets(datasetType=dataset_type_expr,
65 collections=collections_expr,
66 where=where,
67 findFirst=find_first)
69 # Might need expanded results if datastore records have to be derived.
70 # Not all registries return the same form for results.
71 if isinstance(source_refs, DatasetQueryResults):
72 source_refs = source_refs.expanded()
74 # Place results in a set to remove duplicates
75 source_refs = set(source_refs)
77 transferred = dest_butler.transfer_from(source_butler, source_refs, transfer=transfer,
78 register_dataset_types=register_dataset_types)
79 return len(transferred)