Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import copy 

23import unittest 

24import os.path 

25import posixpath 

26import pickle 

27import pathlib 

28 

29from lsst.daf.butler import LocationFactory, Location, ButlerURI 

30from lsst.daf.butler.core._butlerUri.utils import os2posix, posix2os 

31 

32 

33class LocationTestCase(unittest.TestCase): 

34 """Tests for Location within datastore 

35 """ 

36 

37 def testButlerUri(self): 

38 """Tests whether ButlerURI instantiates correctly given different 

39 arguments. 

40 """ 

41 # Root to use for relative paths 

42 testRoot = "/tmp/" 

43 

44 # uriStrings is a list of tuples containing test string, forceAbsolute, 

45 # forceDirectory as arguments to ButlerURI and scheme, netloc and path 

46 # as expected attributes. Test asserts constructed equals to expected. 

47 # 1) no determinable schemes (ensures schema and netloc are not set) 

48 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

49 uriStrings = [ 

50 ("relative/file.ext", True, False, "", "", osRelFilePath), 

51 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

52 ("test/../relative/file.ext", True, False, "", "", osRelFilePath), 

53 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

54 ("relative/dir", False, True, "", "", "relative/dir/") 

55 ] 

56 # 2) implicit file scheme, tests absolute file and directory paths 

57 uriStrings.extend(( 

58 ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'), 

59 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

60 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

61 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

62 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

63 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")) 

64 )) 

65 # 3) explicit file scheme, absolute and relative file and directory URI 

66 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

67 uriStrings.extend(( 

68 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

69 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

70 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

71 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/") 

72 )) 

73 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

74 uriStrings.extend(( 

75 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

76 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

77 ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3", 

78 "bucketname", "/rootDir/relative/file.ext") 

79 )) 

80 # 5) HTTPS scheme 

81 uriStrings.extend(( 

82 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"), 

83 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"), 

84 ("https://www.lsst.org/rootDir/relative/file.ext", True, False, "https", 

85 "www.lsst.org", "/rootDir/relative/file.ext") 

86 )) 

87 

88 for uriInfo in uriStrings: 

89 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], 

90 forceDirectory=uriInfo[2]) 

91 with self.subTest(uri=uriInfo[0]): 

92 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

93 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

94 self.assertEqual(uri.path, uriInfo[5], "test path") 

95 

96 # test root becomes abspath(".") when not specified, note specific 

97 # file:// scheme case 

98 uriStrings = ( 

99 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

100 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

101 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"), 

102 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")) 

103 ) 

104 

105 for uriInfo in uriStrings: 

106 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

107 with self.subTest(uri=uriInfo[0]): 

108 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

109 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

110 self.assertEqual(uri.path, uriInfo[5], "test path") 

111 

112 # File replacement 

113 uriStrings = ( 

114 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

115 ("relative/", "newfile.fits", "relative/newfile.fits"), 

116 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

117 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

118 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml") 

119 ) 

120 

121 for uriInfo in uriStrings: 

122 uri = ButlerURI(uriInfo[0], forceAbsolute=False) 

123 uri.updateFile(uriInfo[1]) 

124 with self.subTest(uri=uriInfo[0]): 

125 self.assertEqual(uri.path, uriInfo[2]) 

126 

127 # Check that schemeless can become file scheme 

128 schemeless = ButlerURI("relative/path.ext") 

129 filescheme = ButlerURI("/absolute/path.ext") 

130 self.assertFalse(schemeless.scheme) 

131 self.assertEqual(filescheme.scheme, "file") 

132 self.assertNotEqual(type(schemeless), type(filescheme)) 

133 

134 # Copy constructor 

135 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True) 

136 uri2 = ButlerURI(uri) 

137 self.assertEqual(uri, uri2) 

138 uri = ButlerURI("file://amazon/datastore/file.txt") 

139 uri2 = ButlerURI(uri) 

140 self.assertEqual(uri, uri2) 

141 

142 # Copy constructor using subclass 

143 uri3 = type(uri)(uri) 

144 self.assertEqual(type(uri), type(uri3)) 

145 

146 # Explicit copy 

147 uri4 = copy.copy(uri3) 

148 self.assertEqual(uri4, uri3) 

149 uri4 = copy.deepcopy(uri3) 

150 self.assertEqual(uri4, uri3) 

151 

152 def testUriRoot(self): 

153 osPathRoot = pathlib.Path(__file__).absolute().root 

154 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com") 

155 for uri_str in rootUris: 

156 uri = ButlerURI(uri_str, forceDirectory=True) 

157 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}") 

158 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI") 

159 

160 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c") 

161 uriStrings = ( 

162 ("file://localhost/file.ext", "file.ext"), 

163 (exampleLocalFile, os.path.join("a", "b", "c")), 

164 ("s3://bucket/path/file.ext", "path/file.ext"), 

165 ("https://host.com/a/b/c.d", "a/b/c.d"), 

166 ) 

167 

168 for uri_str, result in uriStrings: 

169 uri = ButlerURI(uri_str) 

170 self.assertEqual(uri.relativeToPathRoot, result) 

171 

172 def testUriJoin(self): 

173 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False) 

174 uri2 = uri.join("e/f/g.txt") 

175 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

176 

177 uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False) 

178 uri2 = uri.join("e/f/g.txt") 

179 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

180 

181 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True) 

182 uri2 = uri.join("e/f/g.txt") 

183 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") 

184 

185 uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True) 

186 uri2 = uri.join("newpath/newfile.txt") 

187 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

188 

189 uri = ButlerURI("s3://bucket/a/b/c/d/old.txt") 

190 uri2 = uri.join("newpath/newfile.txt") 

191 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

192 

193 def testButlerUriSerialization(self): 

194 """Test that we can pickle and yaml""" 

195 uri = ButlerURI("a/b/c/d") 

196 uri2 = pickle.loads(pickle.dumps(uri)) 

197 self.assertEqual(uri, uri2) 

198 self.assertFalse(uri2.dirLike) 

199 

200 uri = ButlerURI("a/b/c/d", forceDirectory=True) 

201 uri2 = pickle.loads(pickle.dumps(uri)) 

202 self.assertEqual(uri, uri2) 

203 self.assertTrue(uri2.dirLike) 

204 

205 def testUriExtensions(self): 

206 """Test extension extraction.""" 

207 

208 files = (("file.fits.gz", ".fits.gz"), 

209 ("file.fits", ".fits"), 

210 ("file.fits.xz", ".fits.xz"), 

211 ("file.fits.tar", ".tar"), 

212 ("file", ""), 

213 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"), 

214 ("flat_i_sim_1.4_blah.txt", ".txt"), 

215 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"), 

216 ("flat_i_sim_1.4_blah.fits.txt", ".txt"), 

217 ) 

218 

219 for file, expected in files: 

220 uri = ButlerURI(f"a/b/{file}") 

221 self.assertEqual(uri.getExtension(), expected) 

222 

223 def testFileLocation(self): 

224 root = os.path.abspath(os.path.curdir) 

225 factory = LocationFactory(root) 

226 print(f"Factory created: {factory}") 

227 

228 pathInStore = "relative/path/file.ext" 

229 loc1 = factory.fromPath(pathInStore) 

230 

231 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

232 self.assertEqual(loc1.pathInStore.path, pathInStore) 

233 self.assertTrue(loc1.uri.geturl().startswith("file:///")) 

234 self.assertTrue(loc1.uri.geturl().endswith("file.ext")) 

235 loc1.updateExtension("fits") 

236 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), 

237 f"Checking 'fits' extension in {loc1.uri}") 

238 loc1.updateExtension("fits.gz") 

239 self.assertEqual(loc1.uri.basename(), "file.fits.gz") 

240 self.assertTrue(loc1.uri.geturl().endswith("file.fits.gz"), 

241 f"Checking 'fits.gz' extension in {loc1.uri}") 

242 self.assertEqual(loc1.getExtension(), ".fits.gz") 

243 loc1.updateExtension(".jpeg") 

244 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), 

245 f"Checking 'jpeg' extension in {loc1.uri}") 

246 loc1.updateExtension(None) 

247 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), 

248 f"Checking unchanged extension in {loc1.uri}") 

249 loc1.updateExtension("") 

250 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}") 

251 self.assertEqual(loc1.getExtension(), "") 

252 

253 loc2 = factory.fromPath(pathInStore) 

254 loc3 = factory.fromPath(pathInStore) 

255 self.assertEqual(loc2, loc3) 

256 

257 def testAbsoluteLocations(self): 

258 """Using a pathInStore that refers to absolute URI.""" 

259 loc = Location(None, "file:///something.txt") 

260 self.assertEqual(loc.pathInStore.path, "/something.txt") 

261 self.assertEqual(str(loc.uri), "file:///something.txt") 

262 

263 with self.assertRaises(ValueError): 

264 Location(None, "relative.txt") 

265 

266 def testRelativeRoot(self): 

267 root = os.path.abspath(os.path.curdir) 

268 factory = LocationFactory(os.path.curdir) 

269 

270 pathInStore = "relative/path/file.ext" 

271 loc1 = factory.fromPath(pathInStore) 

272 

273 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

274 self.assertEqual(loc1.pathInStore.path, pathInStore) 

275 self.assertEqual(loc1.uri.scheme, "file") 

276 

277 with self.assertRaises(ValueError): 

278 factory.fromPath("../something") 

279 

280 def testQuotedRoot(self): 

281 """Test we can handle quoted characters.""" 

282 root = "/a/b/c+1/d" 

283 factory = LocationFactory(root) 

284 

285 pathInStore = "relative/path/file.ext.gz" 

286 

287 for pathInStore in ("relative/path/file.ext.gz", 

288 "relative/path+2/file.ext.gz", 

289 "relative/path+3/file&.ext.gz"): 

290 loc1 = factory.fromPath(pathInStore) 

291 

292 self.assertEqual(loc1.pathInStore.path, pathInStore) 

293 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

294 self.assertIn("%", str(loc1.uri)) 

295 self.assertEqual(loc1.getExtension(), ".ext.gz") 

296 

297 def testHttpLocation(self): 

298 root = "https://www.lsst.org/butler/datastore" 

299 factory = LocationFactory(root) 

300 print(f"Factory created: {factory}") 

301 

302 pathInStore = "relative/path/file.ext" 

303 loc1 = factory.fromPath(pathInStore) 

304 

305 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

306 self.assertEqual(loc1.pathInStore.path, pathInStore) 

307 self.assertEqual(loc1.uri.scheme, "https") 

308 self.assertEqual(loc1.uri.basename(), "file.ext") 

309 loc1.updateExtension("fits") 

310 self.assertTrue(loc1.uri.basename(), "file.fits") 

311 

312 def testPosix2OS(self): 

313 """Test round tripping of the posix to os.path conversion helpers.""" 

314 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

315 for p in testPaths: 

316 with self.subTest(path=p): 

317 self.assertEqual(os2posix(posix2os(p)), p) 

318 

319 def testSplit(self): 

320 """Tests split functionality.""" 

321 testRoot = "/tmp/" 

322 

323 testPaths = ("/absolute/file.ext", "/absolute/", 

324 "file:///absolute/file.ext", "file:///absolute/", 

325 "s3://bucket/root/file.ext", "s3://bucket/root/", 

326 "https://www.lsst.org/root/file.ext", "https://www.lsst.org/root/", 

327 "relative/file.ext", "relative/") 

328 

329 osRelExpected = os.path.join(testRoot, "relative") 

330 expected = (("file:///absolute/", "file.ext"), ("file:///absolute/", ""), 

331 ("file:///absolute/", "file.ext"), ("file:///absolute/", ""), 

332 ("s3://bucket/root/", "file.ext"), ("s3://bucket/root/", ""), 

333 ("https://www.lsst.org/root/", "file.ext"), ("https://www.lsst.org/root/", ""), 

334 (f"file://{osRelExpected}/", "file.ext"), (f"file://{osRelExpected}/", "")) 

335 

336 for p, e in zip(testPaths, expected): 

337 with self.subTest(path=p): 

338 uri = ButlerURI(p, testRoot) 

339 head, tail = uri.split() 

340 self.assertEqual((head.geturl(), tail), e) 

341 

342 # explicit file scheme should force posixpath, check os.path is ignored 

343 posixRelFilePath = posixpath.join(testRoot, "relative") 

344 uri = ButlerURI("file:relative/file.ext", testRoot) 

345 head, tail = uri.split() 

346 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

347 

348 # check head can be empty and we do not get an absolute path back 

349 uri = ButlerURI("file.ext", forceAbsolute=False) 

350 head, tail = uri.split() 

351 self.assertEqual((head.geturl(), tail), ("./", "file.ext")) 

352 

353 # ensure empty path splits to a directory URL 

354 uri = ButlerURI("", forceAbsolute=False) 

355 head, tail = uri.split() 

356 self.assertEqual((head.geturl(), tail), ("./", "")) 

357 

358 uri = ButlerURI(".", forceAbsolute=False) 

359 head, tail = uri.split() 

360 self.assertEqual((head.geturl(), tail), ("./", "")) 

361 

362 

363if __name__ == "__main__": 363 ↛ 364line 363 didn't jump to line 364, because the condition on line 363 was never true

364 unittest.main()