Coverage for python/lsst/daf/butler/script/retrieveArtifacts.py: 53%

15 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-26 02:47 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("retrieveArtifacts",) 

31 

32import logging 

33from types import EllipsisType 

34from typing import TYPE_CHECKING 

35 

36from .._butler import Butler 

37 

38if TYPE_CHECKING: 

39 from lsst.resources import ResourcePath 

40 

41log = logging.getLogger(__name__) 

42 

43 

44def retrieveArtifacts( 

45 repo: str, 

46 destination: str, 

47 dataset_type: tuple[str, ...], 

48 collections: tuple[str, ...], 

49 where: str, 

50 find_first: bool, 

51 transfer: str, 

52 preserve_path: bool, 

53 clobber: bool, 

54) -> list[ResourcePath]: 

55 """Parameters are those required for querying datasets plus a destination 

56 URI. 

57 

58 Parameters 

59 ---------- 

60 repo : `str` 

61 URI string of the Butler repo to use. 

62 destination : `str` 

63 URI string of the directory to write the artifacts. 

64 dataset_type : `tuple` of `str` 

65 Dataset type names. An empty tuple implies all dataset types. 

66 collections : `tuple` of `str` 

67 Names of collection globs to match. An empty tuple implies all 

68 collections. 

69 where : `str` 

70 Query modification string. 

71 find_first : `bool` 

72 Whether only the first match should be used. 

73 transfer : `str` 

74 Transfer mode to use when placing artifacts in the destination. 

75 preserve_path : `bool` 

76 If `True` the full datastore path will be retained within the 

77 destination directory, else only the filename will be used. 

78 clobber : `bool` 

79 If `True` allow transfers to overwrite files at the destination. 

80 

81 Returns 

82 ------- 

83 transferred : `list` of `lsst.resources.ResourcePath` 

84 The destination URIs of every transferred artifact. 

85 """ 

86 query_types = dataset_type or ... 

87 query_collections: tuple[str, ...] | EllipsisType = collections or ... 

88 

89 butler = Butler.from_config(repo, writeable=False) 

90 

91 # Need to store in list so we can count the number to give some feedback 

92 # to caller. 

93 refs = list( 

94 butler.registry.queryDatasets( 

95 datasetType=query_types, collections=query_collections, where=where, findFirst=find_first 

96 ) 

97 ) 

98 

99 log.info("Number of datasets matching query: %d", len(refs)) 

100 

101 transferred = butler.retrieveArtifacts( 

102 refs, destination=destination, transfer=transfer, preserve_path=preserve_path, overwrite=clobber 

103 ) 

104 return transferred