Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import os.path 

24import posixpath 

25 

26from lsst.daf.butler import LocationFactory, ButlerURI 

27from lsst.daf.butler.core.location import os2posix, posix2os 

28 

29 

30class LocationTestCase(unittest.TestCase): 

31 """Tests for Location within datastore 

32 """ 

33 

34 def testButlerUri(self): 

35 """Tests whether ButlerURI instantiates correctly given different 

36 arguments. 

37 """ 

38 # Root to use for relative paths 

39 testRoot = "/tmp/" 

40 

41 # uriStrings is a list of tuples containing test string, forceAbsolute, 

42 # forceDirectory as arguments to ButlerURI and scheme, netloc and path 

43 # as expected attributes. Test asserts constructed equals to expected. 

44 # 1) no determinable schemes (ensures schema and netloc are not set) 

45 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

46 uriStrings = [ 

47 ("relative/file.ext", True, False, "", "", osRelFilePath), 

48 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

49 ("test/../relative/file.ext", True, False, "", "", osRelFilePath), 

50 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

51 ("relative/dir", False, True, "", "", "relative/dir/") 

52 ] 

53 # 2) implicit file scheme, tests absolute file and directory paths 

54 uriStrings.extend(( 

55 ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'), 

56 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

57 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

58 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

59 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

60 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")) 

61 )) 

62 # 3) explicit file scheme, absolute and relative file and directory URI 

63 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

64 uriStrings.extend(( 

65 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

66 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

67 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

68 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/") 

69 )) 

70 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

71 uriStrings.extend(( 

72 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

73 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

74 ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3", 

75 "bucketname", "/rootDir/relative/file.ext") 

76 )) 

77 

78 for uriInfo in uriStrings: 

79 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], 

80 forceDirectory=uriInfo[2]) 

81 with self.subTest(uri=uriInfo[0]): 

82 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

83 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

84 self.assertEqual(uri.path, uriInfo[5], "test path") 

85 

86 # test root becomes abspath(".") when not specified, note specific 

87 # file:// scheme case 

88 uriStrings = ( 

89 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

90 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

91 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"), 

92 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")) 

93 ) 

94 

95 for uriInfo in uriStrings: 

96 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

97 with self.subTest(uri=uriInfo[0]): 

98 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

99 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

100 self.assertEqual(uri.path, uriInfo[5], "test path") 

101 

102 # File replacement 

103 uriStrings = ( 

104 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

105 ("relative/", "newfile.fits", "relative/newfile.fits"), 

106 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

107 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

108 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml") 

109 ) 

110 

111 for uriInfo in uriStrings: 

112 uri = ButlerURI(uriInfo[0], forceAbsolute=False) 

113 uri.updateFile(uriInfo[1]) 

114 with self.subTest(uri=uriInfo[0]): 

115 self.assertEqual(uri.path, uriInfo[2]) 

116 

117 # Copy constructor 

118 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True) 

119 uri2 = ButlerURI(uri) 

120 self.assertEqual(uri, uri2) 

121 uri = ButlerURI("file://amazon/datastore/file.txt") 

122 uri2 = ButlerURI(uri) 

123 self.assertEqual(uri, uri2) 

124 

125 def testFileLocation(self): 

126 root = os.path.abspath(os.path.curdir) 

127 factory = LocationFactory(root) 

128 print(f"Factory created: {factory}") 

129 

130 pathInStore = "relative/path/file.ext" 

131 loc1 = factory.fromPath(pathInStore) 

132 

133 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

134 self.assertEqual(loc1.pathInStore, pathInStore) 

135 self.assertTrue(loc1.uri.startswith("file:///")) 

136 self.assertTrue(loc1.uri.endswith("file.ext")) 

137 loc1.updateExtension("fits") 

138 self.assertTrue(loc1.uri.endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}") 

139 loc1.updateExtension("fits.gz") 

140 self.assertTrue(loc1.uri.endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}") 

141 loc1.updateExtension(".jpeg") 

142 self.assertTrue(loc1.uri.endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}") 

143 loc1.updateExtension(None) 

144 self.assertTrue(loc1.uri.endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}") 

145 loc1.updateExtension("") 

146 self.assertTrue(loc1.uri.endswith("file"), f"Checking no extension in {loc1.uri}") 

147 

148 def testRelativeRoot(self): 

149 root = os.path.abspath(os.path.curdir) 

150 factory = LocationFactory(os.path.curdir) 

151 print(f"Factory created: {factory}") 

152 

153 pathInStore = "relative/path/file.ext" 

154 loc1 = factory.fromPath(pathInStore) 

155 

156 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

157 self.assertEqual(loc1.pathInStore, pathInStore) 

158 self.assertTrue(loc1.uri.startswith("file:///")) 

159 

160 def testHttpLocation(self): 

161 root = "https://www.lsst.org/butler/datastore" 

162 factory = LocationFactory(root) 

163 print(f"Factory created: {factory}") 

164 

165 pathInStore = "relative/path/file.ext" 

166 loc1 = factory.fromPath(pathInStore) 

167 

168 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

169 self.assertEqual(loc1.pathInStore, pathInStore) 

170 self.assertTrue(loc1.uri.startswith("https://")) 

171 self.assertTrue(loc1.uri.endswith("file.ext")) 

172 loc1.updateExtension("fits") 

173 self.assertTrue(loc1.uri.endswith("file.fits")) 

174 

175 def testPosix2OS(self): 

176 """Test round tripping of the posix to os.path conversion helpers.""" 

177 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

178 for p in testPaths: 

179 with self.subTest(path=p): 

180 self.assertEqual(os2posix(posix2os(p)), p) 

181 

182 def testSplit(self): 

183 """Tests split functionality.""" 

184 testRoot = "/tmp/" 

185 

186 testPaths = ("/absolute/file.ext", "/absolute/", 

187 "file:///absolute/file.ext", "file:///absolute/", 

188 "s3://bucket/root/file.ext", "s3://bucket/root/", 

189 "relative/file.ext", "relative/") 

190 

191 osRelExpected = os.path.join(testRoot, "relative") 

192 expected = (("file:///absolute/", "file.ext"), ("file:///absolute/", ""), 

193 ("file:///absolute/", "file.ext"), ("file:///absolute/", ""), 

194 ("s3://bucket/root/", "file.ext"), ("s3://bucket/root/", ""), 

195 (f"file://{osRelExpected}/", "file.ext"), (f"file://{osRelExpected}/", "")) 

196 

197 for p, e in zip(testPaths, expected): 

198 with self.subTest(path=p): 

199 uri = ButlerURI(p, testRoot) 

200 head, tail = uri.split() 

201 self.assertEqual((head.geturl(), tail), e) 

202 

203 # explicit file scheme should force posixpath, check os.path is ignored 

204 posixRelFilePath = posixpath.join(testRoot, "relative") 

205 uri = ButlerURI("file:relative/file.ext", testRoot) 

206 head, tail = uri.split() 

207 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

208 

209 # check head can be empty 

210 curDir = os.path.abspath(os.path.curdir) + os.sep 

211 uri = ButlerURI("file.ext", forceAbsolute=False) 

212 head, tail = uri.split() 

213 self.assertEqual((head.geturl(), tail), (curDir, "file.ext")) 

214 

215 # ensure empty path is not a problem and conforms to os.path.split 

216 uri = ButlerURI("", forceAbsolute=False) 

217 head, tail = uri.split() 

218 self.assertEqual((head.geturl(), tail), (curDir, "")) 

219 

220 uri = ButlerURI(".", forceAbsolute=False) 

221 head, tail = uri.split() 

222 self.assertEqual((head.geturl(), tail), (curDir, ".")) 

223 

224 

225if __name__ == "__main__": 225 ↛ 226line 225 didn't jump to line 226, because the condition on line 225 was never true

226 unittest.main()