Coverage for tests/test_location.py: 10%

193 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-22 03:00 -0800

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import copy 

13import os.path 

14import pathlib 

15import pickle 

16import posixpath 

17import unittest 

18 

19from lsst.resources import ResourcePath 

20from lsst.resources.location import Location, LocationFactory 

21from lsst.resources.utils import os2posix, posix2os 

22 

23 

24class LocationTestCase(unittest.TestCase): 

25 """Tests for Location within datastore.""" 

26 

27 def testResourcePath(self): 

28 """Tests whether ResourcePath instantiates correctly given different 

29 arguments. 

30 """ 

31 # Root to use for relative paths 

32 testRoot = "/tmp/" 

33 

34 # uriStrings is a list of tuples containing test string, forceAbsolute, 

35 # forceDirectory as arguments to ResourcePath and scheme, netloc and 

36 # path as expected attributes. Test asserts constructed equals to 

37 # expected. 

38 # 1) no determinable schemes (ensures schema and netloc are not set) 

39 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

40 uriStrings = [ 

41 ("relative/file.ext", True, False, "", "", osRelFilePath), 

42 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

43 ("test/../relative/file.ext", True, False, "", "", osRelFilePath), 

44 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

45 ("relative/dir", False, True, "", "", "relative/dir/"), 

46 ] 

47 # 2) implicit file scheme, tests absolute file and directory paths 

48 uriStrings.extend( 

49 ( 

50 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

51 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

52 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

53 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

54 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

55 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")), 

56 ) 

57 ) 

58 # 3) explicit file scheme, absolute and relative file and directory URI 

59 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

60 uriStrings.extend( 

61 ( 

62 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

63 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

64 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

65 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"), 

66 ) 

67 ) 

68 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

69 uriStrings.extend( 

70 ( 

71 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

72 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

73 ( 

74 "s3://bucketname/rootDir/relative/file.ext", 

75 True, 

76 False, 

77 "s3", 

78 "bucketname", 

79 "/rootDir/relative/file.ext", 

80 ), 

81 ) 

82 ) 

83 # 5) HTTPS scheme 

84 uriStrings.extend( 

85 ( 

86 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"), 

87 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"), 

88 ( 

89 "https://www.lsst.org/rootDir/relative/file.ext", 

90 True, 

91 False, 

92 "https", 

93 "www.lsst.org", 

94 "/rootDir/relative/file.ext", 

95 ), 

96 ) 

97 ) 

98 

99 for uriInfo in uriStrings: 

100 uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

101 with self.subTest(uri=uriInfo[0]): 

102 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

103 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

104 self.assertEqual(uri.path, uriInfo[5], "test path") 

105 

106 # test root becomes abspath(".") when not specified, note specific 

107 # file:// scheme case 

108 uriStrings = ( 

109 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

110 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

111 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"), 

112 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")), 

113 ) 

114 

115 for uriInfo in uriStrings: 

116 uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

117 with self.subTest(uri=uriInfo[0]): 

118 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

119 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

120 self.assertEqual(uri.path, uriInfo[5], "test path") 

121 

122 # File replacement 

123 uriStrings = ( 

124 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

125 ("relative/", "newfile.fits", "relative/newfile.fits"), 

126 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

127 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

128 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"), 

129 ) 

130 

131 for uriInfo in uriStrings: 

132 uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1]) 

133 with self.subTest(uri=uriInfo[0]): 

134 self.assertEqual(uri.path, uriInfo[2]) 

135 

136 # Check that schemeless can become file scheme 

137 schemeless = ResourcePath("relative/path.ext") 

138 filescheme = ResourcePath("/absolute/path.ext") 

139 self.assertFalse(schemeless.scheme) 

140 self.assertEqual(filescheme.scheme, "file") 

141 self.assertNotEqual(type(schemeless), type(filescheme)) 

142 

143 # Copy constructor 

144 uri = ResourcePath("s3://amazon/datastore", forceDirectory=True) 

145 uri2 = ResourcePath(uri) 

146 self.assertEqual(uri, uri2) 

147 uri = ResourcePath("file://amazon/datastore/file.txt") 

148 uri2 = ResourcePath(uri) 

149 self.assertEqual(uri, uri2) 

150 

151 # Copy constructor using subclass 

152 uri3 = type(uri)(uri) 

153 self.assertEqual(type(uri), type(uri3)) 

154 

155 # Explicit copy 

156 uri4 = copy.copy(uri3) 

157 self.assertEqual(uri4, uri3) 

158 uri4 = copy.deepcopy(uri3) 

159 self.assertEqual(uri4, uri3) 

160 

161 def testUriRoot(self): 

162 osPathRoot = pathlib.Path(__file__).absolute().root 

163 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com") 

164 for uri_str in rootUris: 

165 uri = ResourcePath(uri_str, forceDirectory=True) 

166 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}") 

167 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI") 

168 

169 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c") 

170 uriStrings = ( 

171 ("file://localhost/file.ext", "file.ext"), 

172 (exampleLocalFile, os.path.join("a", "b", "c")), 

173 ("s3://bucket/path/file.ext", "path/file.ext"), 

174 ("https://host.com/a/b/c.d", "a/b/c.d"), 

175 ) 

176 

177 for uri_str, result in uriStrings: 

178 uri = ResourcePath(uri_str) 

179 self.assertEqual(uri.relativeToPathRoot, result) 

180 

181 def testUriJoin(self): 

182 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=False) 

183 uri2 = uri.join("e/f/g.txt") 

184 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

185 

186 uri = ResourcePath("a/b/c/d/old.txt", forceAbsolute=False) 

187 uri2 = uri.join("e/f/g.txt") 

188 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

189 

190 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=True) 

191 uri2 = uri.join("e/f/g.txt") 

192 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") 

193 

194 uri = ResourcePath("s3://bucket/a/b/c/d", forceDirectory=True) 

195 uri2 = uri.join("newpath/newfile.txt") 

196 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

197 

198 uri = ResourcePath("s3://bucket/a/b/c/d/old.txt") 

199 uri2 = uri.join("newpath/newfile.txt") 

200 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

201 

202 def testResourcePathSerialization(self): 

203 """Test that we can pickle and yaml""" 

204 uri = ResourcePath("a/b/c/d") 

205 uri2 = pickle.loads(pickle.dumps(uri)) 

206 self.assertEqual(uri, uri2) 

207 self.assertFalse(uri2.dirLike) 

208 

209 uri = ResourcePath("a/b/c/d", forceDirectory=True) 

210 uri2 = pickle.loads(pickle.dumps(uri)) 

211 self.assertEqual(uri, uri2) 

212 self.assertTrue(uri2.dirLike) 

213 

214 def testUriExtensions(self): 

215 """Test extension extraction.""" 

216 

217 files = ( 

218 ("file.fits.gz", ".fits.gz"), 

219 ("file.fits", ".fits"), 

220 ("file.fits.xz", ".fits.xz"), 

221 ("file.fits.tar", ".tar"), 

222 ("file", ""), 

223 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"), 

224 ("flat_i_sim_1.4_blah.txt", ".txt"), 

225 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"), 

226 ("flat_i_sim_1.4_blah.fits.txt", ".txt"), 

227 ("s3://bucket/c/a.b/", ""), 

228 ("s3://bucket/c/a.b", ".b"), 

229 ("file://localhost/c/a.b.gz", ".b.gz"), 

230 ) 

231 

232 for file, expected in files: 

233 test_string = file 

234 if ":" not in test_string: 

235 test_string = f"a/b/{test_string}" 

236 uri = ResourcePath(test_string) 

237 self.assertEqual(uri.getExtension(), expected) 

238 

239 def testFileLocation(self): 

240 root = os.path.abspath(os.path.curdir) 

241 factory = LocationFactory(root) 

242 print(f"Factory created: {factory}") 

243 

244 pathInStore = "relative/path/file.ext" 

245 loc1 = factory.fromPath(pathInStore) 

246 

247 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

248 self.assertEqual(loc1.pathInStore.path, pathInStore) 

249 self.assertTrue(loc1.uri.geturl().startswith("file:///")) 

250 self.assertTrue(loc1.uri.geturl().endswith("file.ext")) 

251 loc1.updateExtension("fits") 

252 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}") 

253 loc1.updateExtension("fits.gz") 

254 self.assertEqual(loc1.uri.basename(), "file.fits.gz") 

255 self.assertTrue( 

256 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}" 

257 ) 

258 self.assertEqual(loc1.getExtension(), ".fits.gz") 

259 loc1.updateExtension(".jpeg") 

260 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}") 

261 loc1.updateExtension(None) 

262 self.assertTrue( 

263 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}" 

264 ) 

265 loc1.updateExtension("") 

266 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}") 

267 self.assertEqual(loc1.getExtension(), "") 

268 

269 loc2 = factory.fromPath(pathInStore) 

270 loc3 = factory.fromPath(pathInStore) 

271 self.assertEqual(loc2, loc3) 

272 

273 def testAbsoluteLocations(self): 

274 """Using a pathInStore that refers to absolute URI.""" 

275 loc = Location(None, "file:///something.txt") 

276 self.assertEqual(loc.pathInStore.path, "/something.txt") 

277 self.assertEqual(str(loc.uri), "file:///something.txt") 

278 

279 with self.assertRaises(ValueError): 

280 Location(None, "relative.txt") 

281 

282 def testRelativeRoot(self): 

283 root = os.path.abspath(os.path.curdir) 

284 factory = LocationFactory(os.path.curdir) 

285 

286 pathInStore = "relative/path/file.ext" 

287 loc1 = factory.fromPath(pathInStore) 

288 

289 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

290 self.assertEqual(loc1.pathInStore.path, pathInStore) 

291 self.assertEqual(loc1.uri.scheme, "file") 

292 

293 with self.assertRaises(ValueError): 

294 factory.fromPath("../something") 

295 

296 def testQuotedRoot(self): 

297 """Test we can handle quoted characters.""" 

298 root = "/a/b/c+1/d" 

299 factory = LocationFactory(root) 

300 

301 pathInStore = "relative/path/file.ext.gz" 

302 

303 for pathInStore in ( 

304 "relative/path/file.ext.gz", 

305 "relative/path+2/file.ext.gz", 

306 "relative/path+3/file&.ext.gz", 

307 ): 

308 loc1 = factory.fromPath(pathInStore) 

309 

310 self.assertEqual(loc1.pathInStore.path, pathInStore) 

311 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

312 self.assertIn("%", str(loc1.uri)) 

313 self.assertEqual(loc1.getExtension(), ".ext.gz") 

314 

315 def testHttpLocation(self): 

316 root = "https://www.lsst.org/butler/datastore" 

317 factory = LocationFactory(root) 

318 print(f"Factory created: {factory}") 

319 

320 pathInStore = "relative/path/file.ext" 

321 loc1 = factory.fromPath(pathInStore) 

322 

323 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

324 self.assertEqual(loc1.pathInStore.path, pathInStore) 

325 self.assertEqual(loc1.uri.scheme, "https") 

326 self.assertEqual(loc1.uri.basename(), "file.ext") 

327 loc1.updateExtension("fits") 

328 self.assertTrue(loc1.uri.basename(), "file.fits") 

329 

330 def testPosix2OS(self): 

331 """Test round tripping of the posix to os.path conversion helpers.""" 

332 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

333 for p in testPaths: 

334 with self.subTest(path=p): 

335 self.assertEqual(os2posix(posix2os(p)), p) 

336 

337 def testSplit(self): 

338 """Tests split functionality.""" 

339 testRoot = "/tmp/" 

340 

341 testPaths = ( 

342 "/absolute/file.ext", 

343 "/absolute/", 

344 "file:///absolute/file.ext", 

345 "file:///absolute/", 

346 "s3://bucket/root/file.ext", 

347 "s3://bucket/root/", 

348 "https://www.lsst.org/root/file.ext", 

349 "https://www.lsst.org/root/", 

350 "relative/file.ext", 

351 "relative/", 

352 ) 

353 

354 osRelExpected = os.path.join(testRoot, "relative") 

355 expected = ( 

356 ("file:///absolute/", "file.ext"), 

357 ("file:///absolute/", ""), 

358 ("file:///absolute/", "file.ext"), 

359 ("file:///absolute/", ""), 

360 ("s3://bucket/root/", "file.ext"), 

361 ("s3://bucket/root/", ""), 

362 ("https://www.lsst.org/root/", "file.ext"), 

363 ("https://www.lsst.org/root/", ""), 

364 (f"file://{osRelExpected}/", "file.ext"), 

365 (f"file://{osRelExpected}/", ""), 

366 ) 

367 

368 for p, e in zip(testPaths, expected): 

369 with self.subTest(path=p): 

370 uri = ResourcePath(p, testRoot) 

371 head, tail = uri.split() 

372 self.assertEqual((head.geturl(), tail), e) 

373 

374 # explicit file scheme should force posixpath, check os.path is ignored 

375 posixRelFilePath = posixpath.join(testRoot, "relative") 

376 uri = ResourcePath("file:relative/file.ext", testRoot) 

377 head, tail = uri.split() 

378 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

379 

380 # check head can be empty and we do not get an absolute path back 

381 uri = ResourcePath("file.ext", forceAbsolute=False) 

382 head, tail = uri.split() 

383 self.assertEqual((head.geturl(), tail), ("./", "file.ext")) 

384 

385 # ensure empty path splits to a directory URL 

386 uri = ResourcePath("", forceAbsolute=False) 

387 head, tail = uri.split() 

388 self.assertEqual((head.geturl(), tail), ("./", "")) 

389 

390 uri = ResourcePath(".", forceAbsolute=False) 

391 head, tail = uri.split() 

392 self.assertEqual((head.geturl(), tail), ("./", "")) 

393 

394 

395if __name__ == "__main__": 395 ↛ 396line 395 didn't jump to line 396, because the condition on line 395 was never true

396 unittest.main()