Coverage for tests/test_location.py: 10%

191 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-25 09:29 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import copy 

13import os.path 

14import pathlib 

15import pickle 

16import posixpath 

17import unittest 

18 

19from lsst.resources import ResourcePath 

20from lsst.resources.location import Location, LocationFactory 

21from lsst.resources.utils import os2posix, posix2os 

22 

23 

24class LocationTestCase(unittest.TestCase): 

25 """Tests for Location within datastore.""" 

26 

27 def testResourcePath(self): 

28 """Tests whether ResourcePath instantiates correctly given different 

29 arguments. 

30 """ 

31 # Root to use for relative paths 

32 testRoot = "/tmp/" 

33 

34 # uriStrings is a list of tuples containing test string, forceAbsolute, 

35 # forceDirectory as arguments to ResourcePath and scheme, netloc and 

36 # path as expected attributes. Test asserts constructed equals to 

37 # expected. 

38 # 1) no determinable schemes (ensures schema and netloc are not set) 

39 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

40 uriStrings = [ 

41 ("relative/file.ext", True, False, "file", "", osRelFilePath), 

42 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

43 ("test/../relative/file.ext", True, False, "file", "", osRelFilePath), 

44 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

45 ("relative/dir", False, True, "", "", "relative/dir/"), 

46 ] 

47 # 2) implicit file scheme, tests absolute file and directory paths 

48 uriStrings.extend( 

49 ( 

50 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

51 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

52 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

53 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

54 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

55 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")), 

56 ) 

57 ) 

58 # 3) explicit file scheme, absolute and relative file and directory URI 

59 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

60 uriStrings.extend( 

61 ( 

62 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

63 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

64 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

65 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"), 

66 ) 

67 ) 

68 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

69 uriStrings.extend( 

70 ( 

71 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

72 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

73 ( 

74 "s3://bucketname/rootDir/relative/file.ext", 

75 True, 

76 False, 

77 "s3", 

78 "bucketname", 

79 "/rootDir/relative/file.ext", 

80 ), 

81 ) 

82 ) 

83 # 5) HTTPS scheme 

84 uriStrings.extend( 

85 ( 

86 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"), 

87 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"), 

88 ( 

89 "https://www.lsst.org/rootDir/relative/file.ext", 

90 True, 

91 False, 

92 "https", 

93 "www.lsst.org", 

94 "/rootDir/relative/file.ext", 

95 ), 

96 ) 

97 ) 

98 

99 for uriInfo in uriStrings: 

100 uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

101 with self.subTest(in_uri=uriInfo[0], out_uri=uri): 

102 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

103 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

104 self.assertEqual(uri.path, uriInfo[5], "test path") 

105 

106 # test root becomes abspath(".") when not specified, note specific 

107 # file:// scheme case 

108 uriStrings = ( 

109 # URI, forceAbsolute, forceDirectory, scheme, netloc, path 

110 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

111 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

112 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"), 

113 ("relative/file.ext", True, False, "file", "", os.path.abspath("relative/file.ext")), 

114 ) 

115 

116 for uriInfo in uriStrings: 

117 uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

118 with self.subTest(in_uri=uriInfo[0], out_uri=uri): 

119 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

120 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

121 # Use ospath here to ensure that we have unquoted any 

122 # special characters in the parent directories. 

123 self.assertEqual(uri.ospath, uriInfo[5], "test path") 

124 

125 # File replacement 

126 uriStrings = ( 

127 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

128 ("relative/", "newfile.fits", "relative/newfile.fits"), 

129 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

130 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

131 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"), 

132 ) 

133 

134 for uriInfo in uriStrings: 

135 uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1]) 

136 with self.subTest(in_uri=uriInfo[0], out_uri=uri): 

137 self.assertEqual(uri.path, uriInfo[2]) 

138 

139 # Check that schemeless can become file scheme. 

140 schemeless = ResourcePath("relative/path.ext") 

141 filescheme = ResourcePath("/absolute/path.ext") 

142 self.assertEqual(schemeless.scheme, "file") 

143 self.assertEqual(filescheme.scheme, "file") 

144 self.assertEqual(type(schemeless), type(filescheme)) 

145 

146 # Copy constructor 

147 uri = ResourcePath("s3://amazon/datastore", forceDirectory=True) 

148 uri2 = ResourcePath(uri) 

149 self.assertEqual(uri, uri2) 

150 uri = ResourcePath("file://amazon/datastore/file.txt") 

151 uri2 = ResourcePath(uri) 

152 self.assertEqual(uri, uri2) 

153 

154 # Copy constructor using subclass 

155 uri3 = type(uri)(uri) 

156 self.assertEqual(type(uri), type(uri3)) 

157 

158 # Explicit copy 

159 uri4 = copy.copy(uri3) 

160 self.assertEqual(uri4, uri3) 

161 uri4 = copy.deepcopy(uri3) 

162 self.assertEqual(uri4, uri3) 

163 

164 def testUriRoot(self): 

165 osPathRoot = pathlib.Path(__file__).absolute().root 

166 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com") 

167 for uri_str in rootUris: 

168 uri = ResourcePath(uri_str, forceDirectory=True) 

169 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}") 

170 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI") 

171 

172 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c") 

173 uriStrings = ( 

174 ("file://localhost/file.ext", "file.ext"), 

175 (exampleLocalFile, os.path.join("a", "b", "c")), 

176 ("s3://bucket/path/file.ext", "path/file.ext"), 

177 ("https://host.com/a/b/c.d", "a/b/c.d"), 

178 ) 

179 

180 for uri_str, result in uriStrings: 

181 uri = ResourcePath(uri_str) 

182 self.assertEqual(uri.relativeToPathRoot, result) 

183 

184 def testUriJoin(self): 

185 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=False) 

186 uri2 = uri.join("e/f/g.txt") 

187 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

188 

189 uri = ResourcePath("a/b/c/d/old.txt", forceAbsolute=False) 

190 uri2 = uri.join("e/f/g.txt") 

191 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

192 

193 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=True) 

194 uri2 = uri.join("e/f/g.txt") 

195 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") 

196 

197 uri = ResourcePath("s3://bucket/a/b/c/d", forceDirectory=True) 

198 uri2 = uri.join("newpath/newfile.txt") 

199 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

200 

201 uri = ResourcePath("s3://bucket/a/b/c/d/old.txt") 

202 uri2 = uri.join("newpath/newfile.txt") 

203 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

204 

205 def testResourcePathSerialization(self): 

206 """Test that we can pickle and yaml""" 

207 uri = ResourcePath("a/b/c/d") 

208 uri2 = pickle.loads(pickle.dumps(uri)) 

209 self.assertEqual(uri, uri2) 

210 self.assertFalse(uri2.dirLike) 

211 

212 uri = ResourcePath("a/b/c/d", forceDirectory=True) 

213 uri2 = pickle.loads(pickle.dumps(uri)) 

214 self.assertEqual(uri, uri2) 

215 self.assertTrue(uri2.dirLike) 

216 

217 def testUriExtensions(self): 

218 """Test extension extraction.""" 

219 files = ( 

220 ("file.fits.gz", ".fits.gz"), 

221 ("file.fits", ".fits"), 

222 ("file.fits.xz", ".fits.xz"), 

223 ("file.fits.tar", ".tar"), 

224 ("file", ""), 

225 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"), 

226 ("flat_i_sim_1.4_blah.txt", ".txt"), 

227 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"), 

228 ("flat_i_sim_1.4_blah.fits.txt", ".txt"), 

229 ("s3://bucket/c/a.b/", ""), 

230 ("s3://bucket/c/a.b", ".b"), 

231 ("file://localhost/c/a.b.gz", ".b.gz"), 

232 ) 

233 

234 for file, expected in files: 

235 test_string = file 

236 if ":" not in test_string: 

237 test_string = f"a/b/{test_string}" 

238 uri = ResourcePath(test_string) 

239 self.assertEqual(uri.getExtension(), expected) 

240 

241 def testFileLocation(self): 

242 root = os.path.abspath(os.path.curdir) 

243 factory = LocationFactory(root) 

244 print(f"Factory created: {factory}") 

245 

246 pathInStore = "relative/path/file.ext" 

247 loc1 = factory.fromPath(pathInStore) 

248 

249 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

250 self.assertEqual(loc1.pathInStore.path, pathInStore) 

251 self.assertTrue(loc1.uri.geturl().startswith("file:///")) 

252 self.assertTrue(loc1.uri.geturl().endswith("file.ext")) 

253 loc1.updateExtension("fits") 

254 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}") 

255 loc1.updateExtension("fits.gz") 

256 self.assertEqual(loc1.uri.basename(), "file.fits.gz") 

257 self.assertTrue( 

258 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}" 

259 ) 

260 self.assertEqual(loc1.getExtension(), ".fits.gz") 

261 loc1.updateExtension(".jpeg") 

262 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}") 

263 loc1.updateExtension(None) 

264 self.assertTrue( 

265 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}" 

266 ) 

267 loc1.updateExtension("") 

268 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}") 

269 self.assertEqual(loc1.getExtension(), "") 

270 

271 loc2 = factory.fromPath(pathInStore) 

272 loc3 = factory.fromPath(pathInStore) 

273 self.assertEqual(loc2, loc3) 

274 

275 def testAbsoluteLocations(self): 

276 """Using a pathInStore that refers to absolute URI.""" 

277 loc = Location(None, "file:///something.txt") 

278 self.assertEqual(loc.pathInStore.path, "/something.txt") 

279 self.assertEqual(str(loc.uri), "file:///something.txt") 

280 

281 with self.assertRaises(ValueError): 

282 Location(None, "relative.txt") 

283 

284 def testRelativeRoot(self): 

285 root = os.path.abspath(os.path.curdir) 

286 factory = LocationFactory(os.path.curdir) 

287 

288 pathInStore = "relative/path/file.ext" 

289 loc1 = factory.fromPath(pathInStore) 

290 

291 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

292 self.assertEqual(loc1.pathInStore.path, pathInStore) 

293 self.assertEqual(loc1.uri.scheme, "file") 

294 

295 with self.assertRaises(ValueError): 

296 factory.fromPath("../something") 

297 

298 def testQuotedRoot(self): 

299 """Test we can handle quoted characters.""" 

300 root = "/a/b/c+1/d" 

301 factory = LocationFactory(root) 

302 

303 pathInStore = "relative/path/file.ext.gz" 

304 

305 for pathInStore in ( 

306 "relative/path/file.ext.gz", 

307 "relative/path+2/file.ext.gz", 

308 "relative/path+3/file&.ext.gz", 

309 ): 

310 loc1 = factory.fromPath(pathInStore) 

311 

312 self.assertEqual(loc1.pathInStore.path, pathInStore) 

313 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

314 self.assertIn("%", str(loc1.uri)) 

315 self.assertEqual(loc1.getExtension(), ".ext.gz") 

316 

317 def testHttpLocation(self): 

318 root = "https://www.lsst.org/butler/datastore" 

319 factory = LocationFactory(root) 

320 print(f"Factory created: {factory}") 

321 

322 pathInStore = "relative/path/file.ext" 

323 loc1 = factory.fromPath(pathInStore) 

324 

325 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

326 self.assertEqual(loc1.pathInStore.path, pathInStore) 

327 self.assertEqual(loc1.uri.scheme, "https") 

328 self.assertEqual(loc1.uri.basename(), "file.ext") 

329 loc1.updateExtension("fits") 

330 self.assertTrue(loc1.uri.basename(), "file.fits") 

331 

332 def testPosix2OS(self): 

333 """Test round tripping of the posix to os.path conversion helpers.""" 

334 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

335 for p in testPaths: 

336 with self.subTest(path=p): 

337 self.assertEqual(os2posix(posix2os(p)), p) 

338 

339 def testSplit(self): 

340 """Tests split functionality.""" 

341 testRoot = "/tmp/" 

342 

343 testPaths = ( 

344 "/absolute/file.ext", 

345 "/absolute/", 

346 "file:///absolute/file.ext", 

347 "file:///absolute/", 

348 "s3://bucket/root/file.ext", 

349 "s3://bucket/root/", 

350 "https://www.lsst.org/root/file.ext", 

351 "https://www.lsst.org/root/", 

352 "relative/file.ext", 

353 "relative/", 

354 ) 

355 

356 osRelExpected = os.path.join(testRoot, "relative") 

357 expected = ( 

358 ("file:///absolute/", "file.ext"), 

359 ("file:///absolute/", ""), 

360 ("file:///absolute/", "file.ext"), 

361 ("file:///absolute/", ""), 

362 ("s3://bucket/root/", "file.ext"), 

363 ("s3://bucket/root/", ""), 

364 ("https://www.lsst.org/root/", "file.ext"), 

365 ("https://www.lsst.org/root/", ""), 

366 (f"file://{osRelExpected}/", "file.ext"), 

367 (f"file://{osRelExpected}/", ""), 

368 ) 

369 

370 for p, e in zip(testPaths, expected): 

371 with self.subTest(path=p): 

372 uri = ResourcePath(p, testRoot) 

373 head, tail = uri.split() 

374 self.assertEqual((head.geturl(), tail), e) 

375 

376 # explicit file scheme should force posixpath, check os.path is ignored 

377 posixRelFilePath = posixpath.join(testRoot, "relative") 

378 uri = ResourcePath("file:relative/file.ext", testRoot) 

379 head, tail = uri.split() 

380 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

381 

382 # check head can be empty and we do not get an absolute path back 

383 uri = ResourcePath("file.ext", forceAbsolute=False) 

384 head, tail = uri.split() 

385 self.assertEqual((head.geturl(), tail), ("./", "file.ext")) 

386 

387 # ensure empty path splits to a directory URL 

388 uri = ResourcePath("", forceAbsolute=False) 

389 head, tail = uri.split() 

390 self.assertEqual((head.geturl(), tail), ("./", "")) 

391 

392 uri = ResourcePath(".", forceAbsolute=False) 

393 head, tail = uri.split() 

394 self.assertEqual((head.geturl(), tail), ("./", "")) 

395 

396 

397if __name__ == "__main__": 

398 unittest.main()