Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import copy 

23import unittest 

24import os.path 

25import posixpath 

26import pickle 

27 

28from lsst.daf.butler import LocationFactory, ButlerURI 

29from lsst.daf.butler.core._butlerUri import os2posix, posix2os 

30 

31 

32class LocationTestCase(unittest.TestCase): 

33 """Tests for Location within datastore 

34 """ 

35 

36 def testButlerUri(self): 

37 """Tests whether ButlerURI instantiates correctly given different 

38 arguments. 

39 """ 

40 # Root to use for relative paths 

41 testRoot = "/tmp/" 

42 

43 # uriStrings is a list of tuples containing test string, forceAbsolute, 

44 # forceDirectory as arguments to ButlerURI and scheme, netloc and path 

45 # as expected attributes. Test asserts constructed equals to expected. 

46 # 1) no determinable schemes (ensures schema and netloc are not set) 

47 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

48 uriStrings = [ 

49 ("relative/file.ext", True, False, "", "", osRelFilePath), 

50 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

51 ("test/../relative/file.ext", True, False, "", "", osRelFilePath), 

52 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

53 ("relative/dir", False, True, "", "", "relative/dir/") 

54 ] 

55 # 2) implicit file scheme, tests absolute file and directory paths 

56 uriStrings.extend(( 

57 ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'), 

58 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

59 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

60 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

61 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

62 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")) 

63 )) 

64 # 3) explicit file scheme, absolute and relative file and directory URI 

65 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

66 uriStrings.extend(( 

67 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

68 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

69 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

70 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/") 

71 )) 

72 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

73 uriStrings.extend(( 

74 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

75 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

76 ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3", 

77 "bucketname", "/rootDir/relative/file.ext") 

78 )) 

79 

80 for uriInfo in uriStrings: 

81 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], 

82 forceDirectory=uriInfo[2]) 

83 with self.subTest(uri=uriInfo[0]): 

84 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

85 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

86 self.assertEqual(uri.path, uriInfo[5], "test path") 

87 

88 # test root becomes abspath(".") when not specified, note specific 

89 # file:// scheme case 

90 uriStrings = ( 

91 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

92 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

93 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"), 

94 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")) 

95 ) 

96 

97 for uriInfo in uriStrings: 

98 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

99 with self.subTest(uri=uriInfo[0]): 

100 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

101 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

102 self.assertEqual(uri.path, uriInfo[5], "test path") 

103 

104 # File replacement 

105 uriStrings = ( 

106 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

107 ("relative/", "newfile.fits", "relative/newfile.fits"), 

108 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

109 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

110 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml") 

111 ) 

112 

113 for uriInfo in uriStrings: 

114 uri = ButlerURI(uriInfo[0], forceAbsolute=False) 

115 uri.updateFile(uriInfo[1]) 

116 with self.subTest(uri=uriInfo[0]): 

117 self.assertEqual(uri.path, uriInfo[2]) 

118 

119 # Check that schemeless can become file scheme 

120 schemeless = ButlerURI("relative/path.ext") 

121 filescheme = ButlerURI("/absolute/path.ext") 

122 self.assertFalse(schemeless.scheme) 

123 self.assertEqual(filescheme.scheme, "file") 

124 self.assertNotEqual(type(schemeless), type(filescheme)) 

125 

126 # Copy constructor 

127 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True) 

128 uri2 = ButlerURI(uri) 

129 self.assertEqual(uri, uri2) 

130 uri = ButlerURI("file://amazon/datastore/file.txt") 

131 uri2 = ButlerURI(uri) 

132 self.assertEqual(uri, uri2) 

133 

134 # Copy constructor using subclass 

135 uri3 = type(uri)(uri) 

136 self.assertEqual(type(uri), type(uri3)) 

137 

138 # Explicit copy 

139 uri4 = copy.copy(uri3) 

140 self.assertEqual(uri4, uri3) 

141 uri4 = copy.deepcopy(uri3) 

142 self.assertEqual(uri4, uri3) 

143 

144 def testUriJoin(self): 

145 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False) 

146 uri2 = uri.join("e/f/g.txt") 

147 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

148 

149 uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False) 

150 uri2 = uri.join("e/f/g.txt") 

151 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

152 

153 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True) 

154 uri2 = uri.join("e/f/g.txt") 

155 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") 

156 

157 uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True) 

158 uri2 = uri.join("newpath/newfile.txt") 

159 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

160 

161 uri = ButlerURI("s3://bucket/a/b/c/d/old.txt") 

162 uri2 = uri.join("newpath/newfile.txt") 

163 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

164 

165 def testButlerUriSerialization(self): 

166 """Test that we can pickle and yaml""" 

167 uri = ButlerURI("a/b/c/d") 

168 uri2 = pickle.loads(pickle.dumps(uri)) 

169 self.assertEqual(uri, uri2) 

170 self.assertFalse(uri2.dirLike) 

171 

172 uri = ButlerURI("a/b/c/d", forceDirectory=True) 

173 uri2 = pickle.loads(pickle.dumps(uri)) 

174 self.assertEqual(uri, uri2) 

175 self.assertTrue(uri2.dirLike) 

176 

177 def testFileLocation(self): 

178 root = os.path.abspath(os.path.curdir) 

179 factory = LocationFactory(root) 

180 print(f"Factory created: {factory}") 

181 

182 pathInStore = "relative/path/file.ext" 

183 loc1 = factory.fromPath(pathInStore) 

184 

185 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

186 self.assertEqual(loc1.pathInStore, pathInStore) 

187 self.assertTrue(loc1.uri.geturl().startswith("file:///")) 

188 self.assertTrue(loc1.uri.geturl().endswith("file.ext")) 

189 loc1.updateExtension("fits") 

190 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), 

191 f"Checking 'fits' extension in {loc1.uri}") 

192 loc1.updateExtension("fits.gz") 

193 self.assertEqual(loc1.uri.basename(), "file.fits.gz") 

194 self.assertTrue(loc1.uri.geturl().endswith("file.fits.gz"), 

195 f"Checking 'fits.gz' extension in {loc1.uri}") 

196 self.assertEqual(loc1.getExtension(), ".fits.gz") 

197 loc1.updateExtension(".jpeg") 

198 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), 

199 f"Checking 'jpeg' extension in {loc1.uri}") 

200 loc1.updateExtension(None) 

201 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), 

202 f"Checking unchanged extension in {loc1.uri}") 

203 loc1.updateExtension("") 

204 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}") 

205 self.assertEqual(loc1.getExtension(), "") 

206 

207 def testRelativeRoot(self): 

208 root = os.path.abspath(os.path.curdir) 

209 factory = LocationFactory(os.path.curdir) 

210 

211 pathInStore = "relative/path/file.ext" 

212 loc1 = factory.fromPath(pathInStore) 

213 

214 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

215 self.assertEqual(loc1.pathInStore, pathInStore) 

216 self.assertEqual(loc1.uri.scheme, "file") 

217 

218 def testQuotedRoot(self): 

219 """Test we can handle quoted characters.""" 

220 root = "/a/b/c+1/d" 

221 factory = LocationFactory(root) 

222 

223 pathInStore = "relative/path/file.ext.gz" 

224 

225 for pathInStore in ("relative/path/file.ext.gz", 

226 "relative/path+2/file.ext.gz", 

227 "relative/path+3/file#.ext.gz"): 

228 loc1 = factory.fromPath(pathInStore) 

229 

230 self.assertEqual(loc1.pathInStore, pathInStore) 

231 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

232 self.assertIn("%", str(loc1.uri)) 

233 self.assertEqual(loc1.getExtension(), ".ext.gz") 

234 

235 def testHttpLocation(self): 

236 root = "https://www.lsst.org/butler/datastore" 

237 factory = LocationFactory(root) 

238 print(f"Factory created: {factory}") 

239 

240 pathInStore = "relative/path/file.ext" 

241 loc1 = factory.fromPath(pathInStore) 

242 

243 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

244 self.assertEqual(loc1.pathInStore, pathInStore) 

245 self.assertEqual(loc1.uri.scheme, "https") 

246 self.assertEqual(loc1.uri.basename(), "file.ext") 

247 loc1.updateExtension("fits") 

248 self.assertTrue(loc1.uri.basename(), "file.fits") 

249 

250 def testPosix2OS(self): 

251 """Test round tripping of the posix to os.path conversion helpers.""" 

252 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

253 for p in testPaths: 

254 with self.subTest(path=p): 

255 self.assertEqual(os2posix(posix2os(p)), p) 

256 

257 def testSplit(self): 

258 """Tests split functionality.""" 

259 testRoot = "/tmp/" 

260 

261 testPaths = ("/absolute/file.ext", "/absolute/", 

262 "file:///absolute/file.ext", "file:///absolute/", 

263 "s3://bucket/root/file.ext", "s3://bucket/root/", 

264 "relative/file.ext", "relative/") 

265 

266 osRelExpected = os.path.join(testRoot, "relative") 

267 expected = (("file:///absolute/", "file.ext"), ("file:///absolute/", ""), 

268 ("file:///absolute/", "file.ext"), ("file:///absolute/", ""), 

269 ("s3://bucket/root/", "file.ext"), ("s3://bucket/root/", ""), 

270 (f"file://{osRelExpected}/", "file.ext"), (f"file://{osRelExpected}/", "")) 

271 

272 for p, e in zip(testPaths, expected): 

273 with self.subTest(path=p): 

274 uri = ButlerURI(p, testRoot) 

275 head, tail = uri.split() 

276 self.assertEqual((head.geturl(), tail), e) 

277 

278 # explicit file scheme should force posixpath, check os.path is ignored 

279 posixRelFilePath = posixpath.join(testRoot, "relative") 

280 uri = ButlerURI("file:relative/file.ext", testRoot) 

281 head, tail = uri.split() 

282 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

283 

284 # check head can be empty and we do not get an absolute path back 

285 uri = ButlerURI("file.ext", forceAbsolute=False) 

286 head, tail = uri.split() 

287 self.assertEqual((head.geturl(), tail), ("./", "file.ext")) 

288 

289 # ensure empty path splits to a directory URL 

290 uri = ButlerURI("", forceAbsolute=False) 

291 head, tail = uri.split() 

292 self.assertEqual((head.geturl(), tail), ("./", "")) 

293 

294 uri = ButlerURI(".", forceAbsolute=False) 

295 head, tail = uri.split() 

296 self.assertEqual((head.geturl(), tail), ("./", "")) 

297 

298 

299if __name__ == "__main__": 299 ↛ 300line 299 didn't jump to line 300, because the condition on line 299 was never true

300 unittest.main()