Coverage for tests / test_cliCmdRetrieveArtifacts.py: 27%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for daf_butler CLI retrieve-artifacts command.""" 

29 

30import os 

31import unittest 

32 

33from lsst.daf.butler import StorageClassFactory 

34from lsst.daf.butler.cli.butler import cli 

35from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg 

36from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

37from lsst.resources import ResourcePath 

38 

39TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

40 

41 

42class CliRetrieveArtifactsTest(unittest.TestCase, ButlerTestHelper): 

43 """Test the retrieve-artifacts command-line.""" 

44 

45 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

46 storageClassFactory = StorageClassFactory() 

47 

48 def setUp(self): 

49 self.root = makeTestTempDir(TESTDIR) 

50 self.testRepo = MetricTestRepo(self.root, configFile=self.configFile) 

51 self.enterContext(self.testRepo.butler) 

52 

53 def tearDown(self): 

54 removeTestTempDir(self.root) 

55 

56 @staticmethod 

57 def find_files(root: str) -> list[ResourcePath]: 

58 return list(ResourcePath.findFileResources([root])) 

59 

60 def testRetrieveAll(self): 

61 runner = LogCliRunner() 

62 with runner.isolated_filesystem(): 

63 # When preserving the path the run will be in the directory along 

64 # with a . in the component name. When not preserving paths the 

65 # filename will have an underscore rather than dot. 

66 for counter, (preserve_path, prefix) in enumerate( 

67 ( 

68 ("--preserve-path", "ingest/run/test_metric_comp."), 

69 ("--no-preserve-path", "test_metric_comp_"), 

70 ) 

71 ): 

72 destdir = f"tmp{counter}/" 

73 result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir, preserve_path]) 

74 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

75 self.assertTrue(result.stdout.endswith(": 6\n"), f"Expected 6 got: {result.stdout}") 

76 

77 artifacts = self.find_files(destdir) 

78 artifacts.sort() 

79 self.assertEqual(len(artifacts), 7, f"Expected 7 artifacts including index: {artifacts}") 

80 self.assertIn(f"{destdir}{prefix}", str(artifacts[1])) 

81 

82 def testRetrieveSubset(self): 

83 runner = LogCliRunner() 

84 with runner.isolated_filesystem(): 

85 destdir = "tmp1/" 

86 result = runner.invoke( 

87 cli, 

88 [ 

89 "retrieve-artifacts", 

90 self.root, 

91 destdir, 

92 "--where", 

93 "instrument='DummyCamComp' AND visit=423", 

94 ], 

95 ) 

96 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

97 self.assertTrue(result.stdout.endswith(": 3\n"), f"Expected 3 got: {result.stdout}") 

98 artifacts = self.find_files(destdir) 

99 artifacts.sort() 

100 self.assertEqual(len(artifacts), 4, f"Expected 4 artifacts including index: {artifacts}") 

101 

102 def testRetrieveAsZip(self): 

103 runner = LogCliRunner() 

104 with runner.isolated_filesystem(): 

105 destdir = "tmp1/" 

106 result = runner.invoke( 

107 cli, 

108 [ 

109 "retrieve-artifacts", 

110 self.root, 

111 destdir, 

112 "--where", 

113 "instrument='DummyCamComp' AND visit=423", 

114 "--zip", 

115 ], 

116 ) 

117 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

118 self.assertIn(".zip", result.stdout) 

119 artifacts = self.find_files(destdir) 

120 self.assertEqual(len(artifacts), 1, f"Expected one zip file: {artifacts}") 

121 

122 def testOverwriteLink(self): 

123 runner = LogCliRunner() 

124 with runner.isolated_filesystem(): 

125 destdir = "tmp2/" 

126 # Force hardlink -- if this fails assume that it is because 

127 # hardlinks are not supported (/tmp and TESTDIR are on 

128 # different file systems) and skip the test. There are other 

129 # tests for the command line itself. 

130 result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir, "--transfer", "hardlink"]) 

131 if result.exit_code != 0: 

132 raise unittest.SkipTest( 

133 "hardlink not supported between these directories for this test:" 

134 f" {clickResultMsg(result)}" 

135 ) 

136 

137 # Running again should pass because hard links are the same 

138 # file. 

139 result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir]) 

140 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

141 

142 def testClobber(self): 

143 runner = LogCliRunner() 

144 with runner.isolated_filesystem(): 

145 destdir = "tmp2/" 

146 # Force copy so we can ensure that overwrite tests will trigger. 

147 result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir, "--transfer", "copy"]) 

148 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

149 

150 # Running again should fail 

151 result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir]) 

152 self.assertNotEqual(result.exit_code, 0, clickResultMsg(result)) 

153 

154 # But with clobber should pass 

155 result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir, "--clobber"]) 

156 self.assertEqual(result.exit_code, 0, clickResultMsg(result)) 

157 

158 

159if __name__ == "__main__": 

160 unittest.main()