Coverage for tests / test_interface.py: 36%

1# This file is part of rucio_register 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22 

23import os 

24import shutil 

25import tempfile 

26from unittest.mock import MagicMock, patch 

27 

28from rucio.client.didclient import DIDClient 

29from rucio.client.replicaclient import ReplicaClient 

30from rucio.common.exception import ( 

31 DataIdentifierAlreadyExists, 

32 DataIdentifierNotFound, 

33 FileAlreadyExists, 

34 RucioException, 

35) 

36 

37import lsst.utils.tests 

38from lsst.daf.butler import Butler, DatasetRef, DimensionUniverse 

39 

40# from lsst.daf.butler.registry import DatasetTypeError, MissingCollectionError 

41from lsst.resources import ResourcePath 

42from lsst.rucio.register.data_type import DataType 

43from lsst.rucio.register.rucio_interface import RucioInterface 

44 

45 

46class InterfaceTestCase(lsst.utils.tests.TestCase): 

47 maxDiff = None 

48 

49 def setUp(self): 

50 self.butler_repo = tempfile.mkdtemp() 

51 test_dir = os.path.abspath(os.path.dirname(__file__)) 

52 

53 self.dataset_ref_file = os.path.join(test_dir, "data", "dataset_ref.json") 

54 

55 Butler.makeRepo(self.butler_repo) 

56 

57 data_name = "visitSummary_HSC_y_HSC-Y_318_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.fits" 

58 json_name = "visitSummary_HSC_y_HSC-Y_318_HSC_runs_RC2_w_2023_32_DM-40356_20230814T170253Z.json" 

59 

60 self.data_file = os.path.join(test_dir, "data", data_name) 

61 self.json_file = os.path.join(test_dir, "data", json_name) 

62 

63 self.butler = Butler(self.butler_repo, writeable=True) 

64 self.butler.getURI = MagicMock(return_value=ResourcePath(f"file://{self.data_file}")) 

65 

66 self.rse_root = tempfile.mkdtemp() 

67 

68 # patch __init__ methods 

69 self.rc_init = patch.object(ReplicaClient, "__init__", return_value=None) 

70 self.dc_init = patch.object(DIDClient, "__init__", return_value=None) 

71 self.rc_add_replicas = patch.object(ReplicaClient, "add_replicas", return_value=None) 

72 self.dc_attach_dids = patch.object(DIDClient, "attach_dids", return_value=None) 

73 self.rand = patch("random.randint", return_value=1) 

74 

75 self.mock_rc_init = self.rc_init.start() 

76 self.mock_dc_init = self.dc_init.start() 

77 self.mock_rc_add_replicas = self.rc_add_replicas.start() 

78 self.mock_dc_attach_dids = self.dc_attach_dids.start() 

79 self.mock_rand = self.rand.start() 

80 

81 rucio_rse = "DRR1" 

82 scope = "test" 

83 dtn_url = "root://xrd1:1094//rucio" 

84 self.ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url, DataType.DATA_PRODUCT) 

85 

86 def testInterfaceTestCase(self): 

87 dtn_url = "root://xrd1:1094//rucio" 

88 

89 json_ref = None 

90 with open(self.dataset_ref_file) as f: 

91 json_ref = f.readline() 

92 

93 ref = DatasetRef.from_json(json_ref, DimensionUniverse()) 

94 

95 self.butler.registry.registerDatasetType(ref.datasetType) 

96 cnt = self.ri.register_as_replicas("mydataset", [ref]) 

97 self.assertEqual(cnt, 1) 

98 

99 rb = self.ri._make_dataset_ref_bundle("mydataset", ref) 

100 self.assertEqual(rb.dataset_id, "mydataset") 

101 

102 did = rb.did.model_dump() 

103 self.assertEqual(did["pfn"], f"{dtn_url}{self.data_file}") 

104 self.assertEqual(did["bytes"], 1365120) 

105 self.assertEqual(did["adler32"], "480be4de") 

106 self.assertEqual(did["md5"], "a7ee5c19f5717bcf8d772de202864244") 

107 self.assertEqual(did["name"], self.data_file) 

108 self.assertEqual(did["scope"], "test") 

109 

110 meta = did["meta"] 

111 self.assertEqual(meta["rubin_butler"], DataType.DATA_PRODUCT) 

112 

113 def common(self): 

114 json_ref = None 

115 with open(self.dataset_ref_file) as f: 

116 json_ref = f.readline() 

117 

118 ref = DatasetRef.from_json(json_ref, DimensionUniverse()) 

119 

120 self.butler.registry.registerDatasetType(ref.datasetType) 

121 self.ri.register_as_replicas("mydataset", [ref]) 

122 

123 @patch.object(ReplicaClient, "add_replicas", side_effect=RucioException("failed")) 

124 def testException1TestCase(self, MC1): 

125 self.ri.register_to_dataset = MagicMock(name="register_to_dataset") 

126 with self.assertRaises(Exception): 

127 self.common() 

128 

129 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

130 def testException2TestCase(self, MC1): 

131 with self.assertRaises(Exception): 

132 self.common() 

133 

134 @patch.object(DIDClient, "add_files_to_dataset", side_effect=FileAlreadyExists("failed")) 

135 def testException3TestCase(self, MC1): 

136 self.common() 

137 

138 @patch.object(DIDClient, "add_dataset", return_value=None) 

139 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

140 def testException4TestCase(self, MC1, MC2): 

141 with self.assertRaises(Exception): 

142 self.common() 

143 

144 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

145 def testException5TestCase(self, MC1): 

146 with self.assertRaises(Exception): 

147 self.common() 

148 

149 @patch.object(DIDClient, "add_dataset", side_effect=DataIdentifierAlreadyExists("failed")) 

150 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

151 def testException6TestCase(self, MC1, MC2): 

152 with self.assertRaises(Exception): 

153 self.common() 

154 

155 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

156 def testException7TestCase(self, MC1): 

157 with self.assertRaises(Exception): 

158 self.common() 

159 

160 @patch.object(DIDClient, "add_dataset", side_effect=RucioException("failed")) 

161 @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) 

162 def testException8TestCase(self, MC1, MC2): 

163 with self.assertRaises(Exception): 

164 self.common() 

165 

166 @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) 

167 def testException9Case(self, MC1): 

168 rucio_rse = "DRR1" 

169 scope = "test" 

170 dtn_url = "root://xrd1:1094//rucio" 

171 

172 ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url, DataType.DATA_PRODUCT) 

173 with self.assertRaises(Exception): 

174 ri._add_file_to_dataset_with_retries(None, None) 

175 

176 def tearDown(self): 

177 patch.stopall() 

178 shutil.rmtree(self.butler_repo, ignore_errors=True) 

179 shutil.rmtree(self.rse_root, ignore_errors=True) 

180 

181 

182class MemoryTester(lsst.utils.tests.MemoryTestCase): 

183 pass 

184 

185 

186def setup_module(module): 

187 lsst.utils.tests.init()