Coverage for tests/test_interface.py: 35%

1# This file is part of rucio_register 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22 

23import os 

24import shutil 

25import tempfile 

26from unittest.mock import MagicMock, patch 

27 

28import lsst.utils.tests 

29from lsst.daf.butler import Butler, DatasetRef, DimensionUniverse 

30 

31# from lsst.daf.butler.registry import DatasetTypeError, MissingCollectionError 

32from lsst.resources import ResourcePath 

33from lsst.rucio.register.rucio_interface import RucioInterface 

34from rucio.client.didclient import DIDClient 

35from rucio.client.replicaclient import ReplicaClient 

36from rucio.common.exception import ( 

37 DataIdentifierAlreadyExists, 

38 DataIdentifierNotFound, 

39 FileAlreadyExists, 

40 RucioException, 

41) 

42 

43 

44class InterfaceTestCase(lsst.utils.tests.TestCase): 

45 maxDiff = None 

46 

47 def setUp(self): 

48 self.butler_repo = tempfile.mkdtemp() 

49 test_dir = os.path.abspath(os.path.dirname(__file__)) 

50 

51 self.dataset_ref_file = os.path.join(test_dir, "data", "dataset_ref.json") 

52 

53 Butler.makeRepo(self.butler_repo) 

54 

55 data_name = "visitSummary_HSC_y_HSC-Y_318_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits" 

56 json_name = "visitSummary_HSC_y_HSC-Y_318_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.json" 

57 

58 self.data_file = os.path.join(test_dir, "data", data_name) 

59 self.json_file = os.path.join(test_dir, "data", json_name) 

60 

61 self.butler = Butler(self.butler_repo, writeable=True) 

62 self.butler.getURI = MagicMock(return_value=ResourcePath(f"file://{self.data_file}")) 

63 

64 self.rse_root = tempfile.mkdtemp() 

65 

66 # patch __init__ methods 

67 self.rc_init = patch.object(ReplicaClient, "__init__", return_value=None) 

68 self.dc_init = patch.object(DIDClient, "__init__", return_value=None) 

69 self.rc_add_replicas = patch.object(ReplicaClient, "add_replicas", return_value=None) 

70 self.dc_attach_dids = patch.object(DIDClient, "attach_dids", return_value=None) 

71 self.rand = patch("random.randint", return_value=1) 

72 

73 self.mock_rc_init = self.rc_init.start() 

74 self.mock_dc_init = self.dc_init.start() 

75 self.mock_rc_add_replicas = self.rc_add_replicas.start() 

76 self.mock_dc_attach_dids = self.dc_attach_dids.start() 

77 self.mock_rand = self.rand.start() 

78 

79 rucio_rse = "DRR1" 

80 scope = "test" 

81 dtn_url = "root://xrd1:1094//rucio" 

82 self.ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url) 

83 

84 def testInterfaceTestCase(self): 

85 dtn_url = "root://xrd1:1094//rucio" 

86 

87 json_ref = None 

88 with open(self.dataset_ref_file) as f: 

89 json_ref = f.readline() 

90 

91 ref = DatasetRef.from_json(json_ref, DimensionUniverse()) 

92 

93 self.butler.registry.registerDatasetType(ref.datasetType) 

94 cnt = self.ri.register_as_replicas("mydataset", [ref]) 

95 self.assertEqual(cnt, 1) 

96 

97 rb = self.ri._make_bundle("mydataset", ref) 

98 self.assertEqual(rb.dataset_id, "mydataset") 

99 

100 did = rb.did.model_dump() 

101 self.assertEqual(did["pfn"], f"{dtn_url}{self.data_file}") 

102 self.assertEqual(did["bytes"], 1365120) 

103 self.assertEqual(did["adler32"], "480be4de") 

104 self.assertEqual(did["md5"], "a7ee5c19f5717bcf8d772de202864244") 

105 self.assertEqual(did["name"], self.data_file) 

106 self.assertEqual(did["scope"], "test") 

107 

108 meta = did["meta"] 

109 self.assertEqual(meta["rubin_butler"], 1) 

110 

111 def common(self): 

112 

113 json_ref = None 

114 with open(self.dataset_ref_file) as f: 

115 json_ref = f.readline() 

116 

117 ref = DatasetRef.from_json(json_ref, DimensionUniverse()) 

118 

119 self.butler.registry.registerDatasetType(ref.datasetType) 

120 self.ri.register_as_replicas("mydataset", [ref]) 

121 

122 @patch.object(ReplicaClient, "add_replicas", side_effect=RucioException("failed")) 

123 def testException1TestCase(self, MC1): 

124 self.ri.register_to_dataset = MagicMock(name="register_to_dataset") 

125 with self.assertRaises(Exception): 

126 self.common() 

127 

128 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

129 def testException2TestCase(self, MC1): 

130 with self.assertRaises(Exception): 

131 self.common() 

132 

133 @patch.object(DIDClient, "add_files_to_dataset", side_effect=FileAlreadyExists("failed")) 

134 def testException3TestCase(self, MC1): 

135 self.common() 

136 

137 @patch.object(DIDClient, "add_dataset", return_value=None) 

138 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

139 def testException4TestCase(self, MC1, MC2): 

140 with self.assertRaises(Exception): 

141 self.common() 

142 

143 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

144 def testException5TestCase(self, MC1): 

145 with self.assertRaises(Exception): 

146 self.common() 

147 

148 @patch.object(DIDClient, "add_dataset", side_effect=DataIdentifierAlreadyExists("failed")) 

149 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

150 def testException6TestCase(self, MC1, MC2): 

151 with self.assertRaises(Exception): 

152 self.common() 

153 

154 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

155 def testException7TestCase(self, MC1): 

156 with self.assertRaises(Exception): 

157 self.common() 

158 

159 @patch.object(DIDClient, "add_dataset", side_effect=RucioException("failed")) 

160 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

161 def testException8TestCase(self, MC1, MC2): 

162 with self.assertRaises(Exception): 

163 self.common() 

164 

165 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

166 def testException9Case(self, MC1): 

167 rucio_rse = "DRR1" 

168 scope = "test" 

169 dtn_url = "root://xrd1:1094//rucio" 

170 

171 ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url) 

172 with self.assertRaises(Exception): 

173 ri._add_file_to_dataset_with_retries(None, None) 

174 

175 def tearDown(self): 

176 patch.stopall() 

177 shutil.rmtree(self.butler_repo, ignore_errors=True) 

178 shutil.rmtree(self.rse_root, ignore_errors=True) 

179 

180 

181class MemoryTester(lsst.utils.tests.MemoryTestCase): 

182 pass 

183 

184 

185def setup_module(module): 

186 lsst.utils.tests.init()