Coverage for tests/test_location.py: 10%

191 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-09 09:05 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import copy 

13import os.path 

14import pathlib 

15import pickle 

16import posixpath 

17import unittest 

18 

19from lsst.resources import ResourcePath 

20from lsst.resources.location import Location, LocationFactory 

21from lsst.resources.utils import os2posix, posix2os 

22 

23 

24class LocationTestCase(unittest.TestCase): 

25 """Tests for Location within datastore.""" 

26 

27 def testResourcePath(self): 

28 """Tests whether ResourcePath instantiates correctly given different 

29 arguments. 

30 """ 

31 # Root to use for relative paths 

32 testRoot = "/tmp/" 

33 

34 # uriStrings is a list of tuples containing test string, forceAbsolute, 

35 # forceDirectory as arguments to ResourcePath and scheme, netloc and 

36 # path as expected attributes. Test asserts constructed equals to 

37 # expected. 

38 # 1) no determinable schemes (ensures schema and netloc are not set) 

39 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

40 uriStrings = [ 

41 ("relative/file.ext", True, False, "file", "", osRelFilePath), 

42 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

43 ("test/../relative/file.ext", True, False, "file", "", osRelFilePath), 

44 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

45 ("relative/dir", False, True, "", "", "relative/dir/"), 

46 ] 

47 # 2) implicit file scheme, tests absolute file and directory paths 

48 uriStrings.extend( 

49 ( 

50 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

51 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

52 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

53 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

54 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

55 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")), 

56 ) 

57 ) 

58 # 3) explicit file scheme, absolute and relative file and directory URI 

59 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

60 uriStrings.extend( 

61 ( 

62 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

63 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

64 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

65 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"), 

66 ) 

67 ) 

68 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

69 uriStrings.extend( 

70 ( 

71 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

72 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

73 ( 

74 "s3://bucketname/rootDir/relative/file.ext", 

75 True, 

76 False, 

77 "s3", 

78 "bucketname", 

79 "/rootDir/relative/file.ext", 

80 ), 

81 ) 

82 ) 

83 # 5) HTTPS scheme 

84 uriStrings.extend( 

85 ( 

86 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"), 

87 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"), 

88 ( 

89 "https://www.lsst.org/rootDir/relative/file.ext", 

90 True, 

91 False, 

92 "https", 

93 "www.lsst.org", 

94 "/rootDir/relative/file.ext", 

95 ), 

96 ) 

97 ) 

98 

99 for uriInfo in uriStrings: 

100 uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

101 with self.subTest(in_uri=uriInfo[0], out_uri=uri): 

102 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

103 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

104 self.assertEqual(uri.path, uriInfo[5], "test path") 

105 

106 # test root becomes abspath(".") when not specified, note specific 

107 # file:// scheme case 

108 uriStrings = ( 

109 # URI, forceAbsolute, forceDirectory, scheme, netloc, path 

110 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

111 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

112 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"), 

113 ("relative/file.ext", True, False, "file", "", os.path.abspath("relative/file.ext")), 

114 ) 

115 

116 for uriInfo in uriStrings: 

117 uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

118 with self.subTest(in_uri=uriInfo[0], out_uri=uri): 

119 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

120 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

121 # Use ospath here to ensure that we have unquoted any 

122 # special characters in the parent directories. 

123 self.assertEqual(uri.ospath, uriInfo[5], "test path") 

124 

125 # File replacement 

126 uriStrings = ( 

127 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

128 ("relative/", "newfile.fits", "relative/newfile.fits"), 

129 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

130 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

131 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"), 

132 ) 

133 

134 for uriInfo in uriStrings: 

135 uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1]) 

136 with self.subTest(in_uri=uriInfo[0], out_uri=uri): 

137 self.assertEqual(uri.path, uriInfo[2]) 

138 

139 # Check that schemeless can become file scheme. 

140 schemeless = ResourcePath("relative/path.ext") 

141 filescheme = ResourcePath("/absolute/path.ext") 

142 self.assertEqual(schemeless.scheme, "file") 

143 self.assertEqual(filescheme.scheme, "file") 

144 self.assertEqual(type(schemeless), type(filescheme)) 

145 

146 # Copy constructor 

147 uri = ResourcePath("s3://amazon/datastore", forceDirectory=True) 

148 uri2 = ResourcePath(uri) 

149 self.assertEqual(uri, uri2) 

150 uri = ResourcePath("file://amazon/datastore/file.txt") 

151 uri2 = ResourcePath(uri) 

152 self.assertEqual(uri, uri2) 

153 

154 # Copy constructor using subclass 

155 uri3 = type(uri)(uri) 

156 self.assertEqual(type(uri), type(uri3)) 

157 

158 # Explicit copy 

159 uri4 = copy.copy(uri3) 

160 self.assertEqual(uri4, uri3) 

161 uri4 = copy.deepcopy(uri3) 

162 self.assertEqual(uri4, uri3) 

163 

164 def testUriRoot(self): 

165 osPathRoot = pathlib.Path(__file__).absolute().root 

166 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com") 

167 for uri_str in rootUris: 

168 uri = ResourcePath(uri_str, forceDirectory=True) 

169 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}") 

170 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI") 

171 

172 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c") 

173 uriStrings = ( 

174 ("file://localhost/file.ext", "file.ext"), 

175 (exampleLocalFile, os.path.join("a", "b", "c")), 

176 ("s3://bucket/path/file.ext", "path/file.ext"), 

177 ("https://host.com/a/b/c.d", "a/b/c.d"), 

178 ) 

179 

180 for uri_str, result in uriStrings: 

181 uri = ResourcePath(uri_str) 

182 self.assertEqual(uri.relativeToPathRoot, result) 

183 

184 def testUriJoin(self): 

185 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=False) 

186 uri2 = uri.join("e/f/g.txt") 

187 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

188 

189 uri = ResourcePath("a/b/c/d/old.txt", forceAbsolute=False) 

190 uri2 = uri.join("e/f/g.txt") 

191 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

192 

193 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=True) 

194 uri2 = uri.join("e/f/g.txt") 

195 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") 

196 

197 uri = ResourcePath("s3://bucket/a/b/c/d", forceDirectory=True) 

198 uri2 = uri.join("newpath/newfile.txt") 

199 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

200 

201 uri = ResourcePath("s3://bucket/a/b/c/d/old.txt") 

202 uri2 = uri.join("newpath/newfile.txt") 

203 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

204 

205 def testResourcePathSerialization(self): 

206 """Test that we can pickle and yaml""" 

207 uri = ResourcePath("a/b/c/d") 

208 uri2 = pickle.loads(pickle.dumps(uri)) 

209 self.assertEqual(uri, uri2) 

210 self.assertFalse(uri2.dirLike) 

211 

212 uri = ResourcePath("a/b/c/d", forceDirectory=True) 

213 uri2 = pickle.loads(pickle.dumps(uri)) 

214 self.assertEqual(uri, uri2) 

215 self.assertTrue(uri2.dirLike) 

216 

217 def testUriExtensions(self): 

218 """Test extension extraction.""" 

219 

220 files = ( 

221 ("file.fits.gz", ".fits.gz"), 

222 ("file.fits", ".fits"), 

223 ("file.fits.xz", ".fits.xz"), 

224 ("file.fits.tar", ".tar"), 

225 ("file", ""), 

226 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"), 

227 ("flat_i_sim_1.4_blah.txt", ".txt"), 

228 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"), 

229 ("flat_i_sim_1.4_blah.fits.txt", ".txt"), 

230 ("s3://bucket/c/a.b/", ""), 

231 ("s3://bucket/c/a.b", ".b"), 

232 ("file://localhost/c/a.b.gz", ".b.gz"), 

233 ) 

234 

235 for file, expected in files: 

236 test_string = file 

237 if ":" not in test_string: 

238 test_string = f"a/b/{test_string}" 

239 uri = ResourcePath(test_string) 

240 self.assertEqual(uri.getExtension(), expected) 

241 

242 def testFileLocation(self): 

243 root = os.path.abspath(os.path.curdir) 

244 factory = LocationFactory(root) 

245 print(f"Factory created: {factory}") 

246 

247 pathInStore = "relative/path/file.ext" 

248 loc1 = factory.fromPath(pathInStore) 

249 

250 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

251 self.assertEqual(loc1.pathInStore.path, pathInStore) 

252 self.assertTrue(loc1.uri.geturl().startswith("file:///")) 

253 self.assertTrue(loc1.uri.geturl().endswith("file.ext")) 

254 loc1.updateExtension("fits") 

255 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}") 

256 loc1.updateExtension("fits.gz") 

257 self.assertEqual(loc1.uri.basename(), "file.fits.gz") 

258 self.assertTrue( 

259 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}" 

260 ) 

261 self.assertEqual(loc1.getExtension(), ".fits.gz") 

262 loc1.updateExtension(".jpeg") 

263 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}") 

264 loc1.updateExtension(None) 

265 self.assertTrue( 

266 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}" 

267 ) 

268 loc1.updateExtension("") 

269 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}") 

270 self.assertEqual(loc1.getExtension(), "") 

271 

272 loc2 = factory.fromPath(pathInStore) 

273 loc3 = factory.fromPath(pathInStore) 

274 self.assertEqual(loc2, loc3) 

275 

276 def testAbsoluteLocations(self): 

277 """Using a pathInStore that refers to absolute URI.""" 

278 loc = Location(None, "file:///something.txt") 

279 self.assertEqual(loc.pathInStore.path, "/something.txt") 

280 self.assertEqual(str(loc.uri), "file:///something.txt") 

281 

282 with self.assertRaises(ValueError): 

283 Location(None, "relative.txt") 

284 

285 def testRelativeRoot(self): 

286 root = os.path.abspath(os.path.curdir) 

287 factory = LocationFactory(os.path.curdir) 

288 

289 pathInStore = "relative/path/file.ext" 

290 loc1 = factory.fromPath(pathInStore) 

291 

292 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

293 self.assertEqual(loc1.pathInStore.path, pathInStore) 

294 self.assertEqual(loc1.uri.scheme, "file") 

295 

296 with self.assertRaises(ValueError): 

297 factory.fromPath("../something") 

298 

299 def testQuotedRoot(self): 

300 """Test we can handle quoted characters.""" 

301 root = "/a/b/c+1/d" 

302 factory = LocationFactory(root) 

303 

304 pathInStore = "relative/path/file.ext.gz" 

305 

306 for pathInStore in ( 

307 "relative/path/file.ext.gz", 

308 "relative/path+2/file.ext.gz", 

309 "relative/path+3/file&.ext.gz", 

310 ): 

311 loc1 = factory.fromPath(pathInStore) 

312 

313 self.assertEqual(loc1.pathInStore.path, pathInStore) 

314 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

315 self.assertIn("%", str(loc1.uri)) 

316 self.assertEqual(loc1.getExtension(), ".ext.gz") 

317 

318 def testHttpLocation(self): 

319 root = "https://www.lsst.org/butler/datastore" 

320 factory = LocationFactory(root) 

321 print(f"Factory created: {factory}") 

322 

323 pathInStore = "relative/path/file.ext" 

324 loc1 = factory.fromPath(pathInStore) 

325 

326 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

327 self.assertEqual(loc1.pathInStore.path, pathInStore) 

328 self.assertEqual(loc1.uri.scheme, "https") 

329 self.assertEqual(loc1.uri.basename(), "file.ext") 

330 loc1.updateExtension("fits") 

331 self.assertTrue(loc1.uri.basename(), "file.fits") 

332 

333 def testPosix2OS(self): 

334 """Test round tripping of the posix to os.path conversion helpers.""" 

335 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

336 for p in testPaths: 

337 with self.subTest(path=p): 

338 self.assertEqual(os2posix(posix2os(p)), p) 

339 

340 def testSplit(self): 

341 """Tests split functionality.""" 

342 testRoot = "/tmp/" 

343 

344 testPaths = ( 

345 "/absolute/file.ext", 

346 "/absolute/", 

347 "file:///absolute/file.ext", 

348 "file:///absolute/", 

349 "s3://bucket/root/file.ext", 

350 "s3://bucket/root/", 

351 "https://www.lsst.org/root/file.ext", 

352 "https://www.lsst.org/root/", 

353 "relative/file.ext", 

354 "relative/", 

355 ) 

356 

357 osRelExpected = os.path.join(testRoot, "relative") 

358 expected = ( 

359 ("file:///absolute/", "file.ext"), 

360 ("file:///absolute/", ""), 

361 ("file:///absolute/", "file.ext"), 

362 ("file:///absolute/", ""), 

363 ("s3://bucket/root/", "file.ext"), 

364 ("s3://bucket/root/", ""), 

365 ("https://www.lsst.org/root/", "file.ext"), 

366 ("https://www.lsst.org/root/", ""), 

367 (f"file://{osRelExpected}/", "file.ext"), 

368 (f"file://{osRelExpected}/", ""), 

369 ) 

370 

371 for p, e in zip(testPaths, expected): 

372 with self.subTest(path=p): 

373 uri = ResourcePath(p, testRoot) 

374 head, tail = uri.split() 

375 self.assertEqual((head.geturl(), tail), e) 

376 

377 # explicit file scheme should force posixpath, check os.path is ignored 

378 posixRelFilePath = posixpath.join(testRoot, "relative") 

379 uri = ResourcePath("file:relative/file.ext", testRoot) 

380 head, tail = uri.split() 

381 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

382 

383 # check head can be empty and we do not get an absolute path back 

384 uri = ResourcePath("file.ext", forceAbsolute=False) 

385 head, tail = uri.split() 

386 self.assertEqual((head.geturl(), tail), ("./", "file.ext")) 

387 

388 # ensure empty path splits to a directory URL 

389 uri = ResourcePath("", forceAbsolute=False) 

390 head, tail = uri.split() 

391 self.assertEqual((head.geturl(), tail), ("./", "")) 

392 

393 uri = ResourcePath(".", forceAbsolute=False) 

394 head, tail = uri.split() 

395 self.assertEqual((head.geturl(), tail), ("./", "")) 

396 

397 

398if __name__ == "__main__": 

399 unittest.main()