Coverage for tests/test_interface.py: 35%

1# This file is part of rucio_register 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22 

23import os 

24import shutil 

25import tempfile 

26from unittest.mock import MagicMock, patch 

27 

28from rucio.client.didclient import DIDClient 

29from rucio.client.replicaclient import ReplicaClient 

30from rucio.common.exception import ( 

31 DataIdentifierAlreadyExists, 

32 DataIdentifierNotFound, 

33 FileAlreadyExists, 

34 RucioException, 

35) 

36 

37import lsst.utils.tests 

38from lsst.daf.butler import Butler, DatasetRef, DimensionUniverse 

39 

40# from lsst.daf.butler.registry import DatasetTypeError, MissingCollectionError 

41from lsst.resources import ResourcePath 

42from lsst.rucio.register.rucio_interface import RucioInterface 

43 

44 

45class InterfaceTestCase(lsst.utils.tests.TestCase): 

46 maxDiff = None 

47 

48 def setUp(self): 

49 self.butler_repo = tempfile.mkdtemp() 

50 test_dir = os.path.abspath(os.path.dirname(__file__)) 

51 

52 self.dataset_ref_file = os.path.join(test_dir, "data", "dataset_ref.json") 

53 

54 Butler.makeRepo(self.butler_repo) 

55 

56 data_name = "visitSummary_HSC_y_HSC-Y_318_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits" 

57 json_name = "visitSummary_HSC_y_HSC-Y_318_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.json" 

58 

59 self.data_file = os.path.join(test_dir, "data", data_name) 

60 self.json_file = os.path.join(test_dir, "data", json_name) 

61 

62 self.butler = Butler(self.butler_repo, writeable=True) 

63 self.butler.getURI = MagicMock(return_value=ResourcePath(f"file://{self.data_file}")) 

64 

65 self.rse_root = tempfile.mkdtemp() 

66 

67 # patch __init__ methods 

68 self.rc_init = patch.object(ReplicaClient, "__init__", return_value=None) 

69 self.dc_init = patch.object(DIDClient, "__init__", return_value=None) 

70 self.rc_add_replicas = patch.object(ReplicaClient, "add_replicas", return_value=None) 

71 self.dc_attach_dids = patch.object(DIDClient, "attach_dids", return_value=None) 

72 self.rand = patch("random.randint", return_value=1) 

73 

74 self.mock_rc_init = self.rc_init.start() 

75 self.mock_dc_init = self.dc_init.start() 

76 self.mock_rc_add_replicas = self.rc_add_replicas.start() 

77 self.mock_dc_attach_dids = self.dc_attach_dids.start() 

78 self.mock_rand = self.rand.start() 

79 

80 rucio_rse = "DRR1" 

81 scope = "test" 

82 dtn_url = "root://xrd1:1094//rucio" 

83 self.ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url) 

84 

85 def testInterfaceTestCase(self): 

86 dtn_url = "root://xrd1:1094//rucio" 

87 

88 json_ref = None 

89 with open(self.dataset_ref_file) as f: 

90 json_ref = f.readline() 

91 

92 ref = DatasetRef.from_json(json_ref, DimensionUniverse()) 

93 

94 self.butler.registry.registerDatasetType(ref.datasetType) 

95 cnt = self.ri.register_as_replicas("mydataset", [ref]) 

96 self.assertEqual(cnt, 1) 

97 

98 rb = self.ri._make_bundle("mydataset", ref) 

99 self.assertEqual(rb.dataset_id, "mydataset") 

100 

101 did = rb.did.model_dump() 

102 self.assertEqual(did["pfn"], f"{dtn_url}{self.data_file}") 

103 self.assertEqual(did["bytes"], 1365120) 

104 self.assertEqual(did["adler32"], "480be4de") 

105 self.assertEqual(did["md5"], "a7ee5c19f5717bcf8d772de202864244") 

106 self.assertEqual(did["name"], self.data_file) 

107 self.assertEqual(did["scope"], "test") 

108 

109 meta = did["meta"] 

110 self.assertEqual(meta["rubin_butler"], 1) 

111 

112 def common(self): 

113 json_ref = None 

114 with open(self.dataset_ref_file) as f: 

115 json_ref = f.readline() 

116 

117 ref = DatasetRef.from_json(json_ref, DimensionUniverse()) 

118 

119 self.butler.registry.registerDatasetType(ref.datasetType) 

120 self.ri.register_as_replicas("mydataset", [ref]) 

121 

122 @patch.object(ReplicaClient, "add_replicas", side_effect=RucioException("failed")) 

123 def testException1TestCase(self, MC1): 

124 self.ri.register_to_dataset = MagicMock(name="register_to_dataset") 

125 with self.assertRaises(Exception): 

126 self.common() 

127 

128 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

129 def testException2TestCase(self, MC1): 

130 with self.assertRaises(Exception): 

131 self.common() 

132 

133 @patch.object(DIDClient, "add_files_to_dataset", side_effect=FileAlreadyExists("failed")) 

134 def testException3TestCase(self, MC1): 

135 self.common() 

136 

137 @patch.object(DIDClient, "add_dataset", return_value=None) 

138 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

139 def testException4TestCase(self, MC1, MC2): 

140 with self.assertRaises(Exception): 

141 self.common() 

142 

143 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

144 def testException5TestCase(self, MC1): 

145 with self.assertRaises(Exception): 

146 self.common() 

147 

148 @patch.object(DIDClient, "add_dataset", side_effect=DataIdentifierAlreadyExists("failed")) 

149 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

150 def testException6TestCase(self, MC1, MC2): 

151 with self.assertRaises(Exception): 

152 self.common() 

153 

154 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

155 def testException7TestCase(self, MC1): 

156 with self.assertRaises(Exception): 

157 self.common() 

158 

159 @patch.object(DIDClient, "add_dataset", side_effect=RucioException("failed")) 

160 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

161 def testException8TestCase(self, MC1, MC2): 

162 with self.assertRaises(Exception): 

163 self.common() 

164 

165 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

166 def testException9Case(self, MC1): 

167 rucio_rse = "DRR1" 

168 scope = "test" 

169 dtn_url = "root://xrd1:1094//rucio" 

170 

171 ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url) 

172 with self.assertRaises(Exception): 

173 ri._add_file_to_dataset_with_retries(None, None) 

174 

175 def tearDown(self): 

176 patch.stopall() 

177 shutil.rmtree(self.butler_repo, ignore_errors=True) 

178 shutil.rmtree(self.rse_root, ignore_errors=True) 

179 

180 

181class MemoryTester(lsst.utils.tests.MemoryTestCase): 

182 pass 

183 

184 

185def setup_module(module): 

186 lsst.utils.tests.init()