Coverage for tests/test_location.py: 12%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

192 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import copy 

23import os.path 

24import pathlib 

25import pickle 

26import posixpath 

27import unittest 

28 

29from lsst.daf.butler import ButlerURI, Location, LocationFactory 

30from lsst.resources.utils import os2posix, posix2os 

31 

32 

33class LocationTestCase(unittest.TestCase): 

34 """Tests for Location within datastore""" 

35 

36 def testButlerUri(self): 

37 """Tests whether ButlerURI instantiates correctly given different 

38 arguments. 

39 """ 

40 # Root to use for relative paths 

41 testRoot = "/tmp/" 

42 

43 # uriStrings is a list of tuples containing test string, forceAbsolute, 

44 # forceDirectory as arguments to ButlerURI and scheme, netloc and path 

45 # as expected attributes. Test asserts constructed equals to expected. 

46 # 1) no determinable schemes (ensures schema and netloc are not set) 

47 osRelFilePath = os.path.join(testRoot, "relative/file.ext") 

48 uriStrings = [ 

49 ("relative/file.ext", True, False, "", "", osRelFilePath), 

50 ("relative/file.ext", False, False, "", "", "relative/file.ext"), 

51 ("test/../relative/file.ext", True, False, "", "", osRelFilePath), 

52 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"), 

53 ("relative/dir", False, True, "", "", "relative/dir/"), 

54 ] 

55 # 2) implicit file scheme, tests absolute file and directory paths 

56 uriStrings.extend( 

57 ( 

58 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

59 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

60 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")), 

61 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"), 

62 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"), 

63 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")), 

64 ) 

65 ) 

66 # 3) explicit file scheme, absolute and relative file and directory URI 

67 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext") 

68 uriStrings.extend( 

69 ( 

70 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"), 

71 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath), 

72 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"), 

73 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"), 

74 ) 

75 ) 

76 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work) 

77 uriStrings.extend( 

78 ( 

79 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"), 

80 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"), 

81 ( 

82 "s3://bucketname/rootDir/relative/file.ext", 

83 True, 

84 False, 

85 "s3", 

86 "bucketname", 

87 "/rootDir/relative/file.ext", 

88 ), 

89 ) 

90 ) 

91 # 5) HTTPS scheme 

92 uriStrings.extend( 

93 ( 

94 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"), 

95 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"), 

96 ( 

97 "https://www.lsst.org/rootDir/relative/file.ext", 

98 True, 

99 False, 

100 "https", 

101 "www.lsst.org", 

102 "/rootDir/relative/file.ext", 

103 ), 

104 ) 

105 ) 

106 

107 for uriInfo in uriStrings: 

108 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

109 with self.subTest(uri=uriInfo[0]): 

110 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

111 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

112 self.assertEqual(uri.path, uriInfo[5], "test path") 

113 

114 # test root becomes abspath(".") when not specified, note specific 

115 # file:// scheme case 

116 uriStrings = ( 

117 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"), 

118 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")), 

119 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"), 

120 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")), 

121 ) 

122 

123 for uriInfo in uriStrings: 

124 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2]) 

125 with self.subTest(uri=uriInfo[0]): 

126 self.assertEqual(uri.scheme, uriInfo[3], "test scheme") 

127 self.assertEqual(uri.netloc, uriInfo[4], "test netloc") 

128 self.assertEqual(uri.path, uriInfo[5], "test path") 

129 

130 # File replacement 

131 uriStrings = ( 

132 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"), 

133 ("relative/", "newfile.fits", "relative/newfile.fits"), 

134 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"), 

135 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"), 

136 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"), 

137 ) 

138 

139 for uriInfo in uriStrings: 

140 uri = ButlerURI(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1]) 

141 with self.subTest(uri=uriInfo[0]): 

142 self.assertEqual(uri.path, uriInfo[2]) 

143 

144 # Check that schemeless can become file scheme 

145 schemeless = ButlerURI("relative/path.ext") 

146 filescheme = ButlerURI("/absolute/path.ext") 

147 self.assertFalse(schemeless.scheme) 

148 self.assertEqual(filescheme.scheme, "file") 

149 self.assertNotEqual(type(schemeless), type(filescheme)) 

150 

151 # Copy constructor 

152 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True) 

153 uri2 = ButlerURI(uri) 

154 self.assertEqual(uri, uri2) 

155 uri = ButlerURI("file://amazon/datastore/file.txt") 

156 uri2 = ButlerURI(uri) 

157 self.assertEqual(uri, uri2) 

158 

159 # Copy constructor using subclass 

160 uri3 = type(uri)(uri) 

161 self.assertEqual(type(uri), type(uri3)) 

162 

163 # Explicit copy 

164 uri4 = copy.copy(uri3) 

165 self.assertEqual(uri4, uri3) 

166 uri4 = copy.deepcopy(uri3) 

167 self.assertEqual(uri4, uri3) 

168 

169 def testUriRoot(self): 

170 osPathRoot = pathlib.Path(__file__).absolute().root 

171 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com") 

172 for uri_str in rootUris: 

173 uri = ButlerURI(uri_str, forceDirectory=True) 

174 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}") 

175 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI") 

176 

177 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c") 

178 uriStrings = ( 

179 ("file://localhost/file.ext", "file.ext"), 

180 (exampleLocalFile, os.path.join("a", "b", "c")), 

181 ("s3://bucket/path/file.ext", "path/file.ext"), 

182 ("https://host.com/a/b/c.d", "a/b/c.d"), 

183 ) 

184 

185 for uri_str, result in uriStrings: 

186 uri = ButlerURI(uri_str) 

187 self.assertEqual(uri.relativeToPathRoot, result) 

188 

189 def testUriJoin(self): 

190 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False) 

191 uri2 = uri.join("e/f/g.txt") 

192 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

193 

194 uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False) 

195 uri2 = uri.join("e/f/g.txt") 

196 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}") 

197 

198 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True) 

199 uri2 = uri.join("e/f/g.txt") 

200 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}") 

201 

202 uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True) 

203 uri2 = uri.join("newpath/newfile.txt") 

204 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

205 

206 uri = ButlerURI("s3://bucket/a/b/c/d/old.txt") 

207 uri2 = uri.join("newpath/newfile.txt") 

208 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt") 

209 

210 def testButlerUriSerialization(self): 

211 """Test that we can pickle and yaml""" 

212 uri = ButlerURI("a/b/c/d") 

213 uri2 = pickle.loads(pickle.dumps(uri)) 

214 self.assertEqual(uri, uri2) 

215 self.assertFalse(uri2.dirLike) 

216 

217 uri = ButlerURI("a/b/c/d", forceDirectory=True) 

218 uri2 = pickle.loads(pickle.dumps(uri)) 

219 self.assertEqual(uri, uri2) 

220 self.assertTrue(uri2.dirLike) 

221 

222 def testUriExtensions(self): 

223 """Test extension extraction.""" 

224 

225 files = ( 

226 ("file.fits.gz", ".fits.gz"), 

227 ("file.fits", ".fits"), 

228 ("file.fits.xz", ".fits.xz"), 

229 ("file.fits.tar", ".tar"), 

230 ("file", ""), 

231 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"), 

232 ("flat_i_sim_1.4_blah.txt", ".txt"), 

233 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"), 

234 ("flat_i_sim_1.4_blah.fits.txt", ".txt"), 

235 ("s3://bucket/c/a.b/", ""), 

236 ("s3://bucket/c/a.b", ".b"), 

237 ("file://localhost/c/a.b.gz", ".b.gz"), 

238 ) 

239 

240 for file, expected in files: 

241 test_string = file 

242 if ":" not in test_string: 

243 test_string = f"a/b/{test_string}" 

244 uri = ButlerURI(test_string) 

245 self.assertEqual(uri.getExtension(), expected) 

246 

247 def testFileLocation(self): 

248 root = os.path.abspath(os.path.curdir) 

249 factory = LocationFactory(root) 

250 print(f"Factory created: {factory}") 

251 

252 pathInStore = "relative/path/file.ext" 

253 loc1 = factory.fromPath(pathInStore) 

254 

255 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

256 self.assertEqual(loc1.pathInStore.path, pathInStore) 

257 self.assertTrue(loc1.uri.geturl().startswith("file:///")) 

258 self.assertTrue(loc1.uri.geturl().endswith("file.ext")) 

259 loc1.updateExtension("fits") 

260 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}") 

261 loc1.updateExtension("fits.gz") 

262 self.assertEqual(loc1.uri.basename(), "file.fits.gz") 

263 self.assertTrue( 

264 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}" 

265 ) 

266 self.assertEqual(loc1.getExtension(), ".fits.gz") 

267 loc1.updateExtension(".jpeg") 

268 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}") 

269 loc1.updateExtension(None) 

270 self.assertTrue( 

271 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}" 

272 ) 

273 loc1.updateExtension("") 

274 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}") 

275 self.assertEqual(loc1.getExtension(), "") 

276 

277 loc2 = factory.fromPath(pathInStore) 

278 loc3 = factory.fromPath(pathInStore) 

279 self.assertEqual(loc2, loc3) 

280 

281 def testAbsoluteLocations(self): 

282 """Using a pathInStore that refers to absolute URI.""" 

283 loc = Location(None, "file:///something.txt") 

284 self.assertEqual(loc.pathInStore.path, "/something.txt") 

285 self.assertEqual(str(loc.uri), "file:///something.txt") 

286 

287 with self.assertRaises(ValueError): 

288 Location(None, "relative.txt") 

289 

290 def testRelativeRoot(self): 

291 root = os.path.abspath(os.path.curdir) 

292 factory = LocationFactory(os.path.curdir) 

293 

294 pathInStore = "relative/path/file.ext" 

295 loc1 = factory.fromPath(pathInStore) 

296 

297 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

298 self.assertEqual(loc1.pathInStore.path, pathInStore) 

299 self.assertEqual(loc1.uri.scheme, "file") 

300 

301 with self.assertRaises(ValueError): 

302 factory.fromPath("../something") 

303 

304 def testQuotedRoot(self): 

305 """Test we can handle quoted characters.""" 

306 root = "/a/b/c+1/d" 

307 factory = LocationFactory(root) 

308 

309 pathInStore = "relative/path/file.ext.gz" 

310 

311 for pathInStore in ( 

312 "relative/path/file.ext.gz", 

313 "relative/path+2/file.ext.gz", 

314 "relative/path+3/file&.ext.gz", 

315 ): 

316 loc1 = factory.fromPath(pathInStore) 

317 

318 self.assertEqual(loc1.pathInStore.path, pathInStore) 

319 self.assertEqual(loc1.path, os.path.join(root, pathInStore)) 

320 self.assertIn("%", str(loc1.uri)) 

321 self.assertEqual(loc1.getExtension(), ".ext.gz") 

322 

323 def testHttpLocation(self): 

324 root = "https://www.lsst.org/butler/datastore" 

325 factory = LocationFactory(root) 

326 print(f"Factory created: {factory}") 

327 

328 pathInStore = "relative/path/file.ext" 

329 loc1 = factory.fromPath(pathInStore) 

330 

331 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore)) 

332 self.assertEqual(loc1.pathInStore.path, pathInStore) 

333 self.assertEqual(loc1.uri.scheme, "https") 

334 self.assertEqual(loc1.uri.basename(), "file.ext") 

335 loc1.updateExtension("fits") 

336 self.assertTrue(loc1.uri.basename(), "file.fits") 

337 

338 def testPosix2OS(self): 

339 """Test round tripping of the posix to os.path conversion helpers.""" 

340 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e") 

341 for p in testPaths: 

342 with self.subTest(path=p): 

343 self.assertEqual(os2posix(posix2os(p)), p) 

344 

345 def testSplit(self): 

346 """Tests split functionality.""" 

347 testRoot = "/tmp/" 

348 

349 testPaths = ( 

350 "/absolute/file.ext", 

351 "/absolute/", 

352 "file:///absolute/file.ext", 

353 "file:///absolute/", 

354 "s3://bucket/root/file.ext", 

355 "s3://bucket/root/", 

356 "https://www.lsst.org/root/file.ext", 

357 "https://www.lsst.org/root/", 

358 "relative/file.ext", 

359 "relative/", 

360 ) 

361 

362 osRelExpected = os.path.join(testRoot, "relative") 

363 expected = ( 

364 ("file:///absolute/", "file.ext"), 

365 ("file:///absolute/", ""), 

366 ("file:///absolute/", "file.ext"), 

367 ("file:///absolute/", ""), 

368 ("s3://bucket/root/", "file.ext"), 

369 ("s3://bucket/root/", ""), 

370 ("https://www.lsst.org/root/", "file.ext"), 

371 ("https://www.lsst.org/root/", ""), 

372 (f"file://{osRelExpected}/", "file.ext"), 

373 (f"file://{osRelExpected}/", ""), 

374 ) 

375 

376 for p, e in zip(testPaths, expected): 

377 with self.subTest(path=p): 

378 uri = ButlerURI(p, testRoot) 

379 head, tail = uri.split() 

380 self.assertEqual((head.geturl(), tail), e) 

381 

382 # explicit file scheme should force posixpath, check os.path is ignored 

383 posixRelFilePath = posixpath.join(testRoot, "relative") 

384 uri = ButlerURI("file:relative/file.ext", testRoot) 

385 head, tail = uri.split() 

386 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext")) 

387 

388 # check head can be empty and we do not get an absolute path back 

389 uri = ButlerURI("file.ext", forceAbsolute=False) 

390 head, tail = uri.split() 

391 self.assertEqual((head.geturl(), tail), ("./", "file.ext")) 

392 

393 # ensure empty path splits to a directory URL 

394 uri = ButlerURI("", forceAbsolute=False) 

395 head, tail = uri.split() 

396 self.assertEqual((head.geturl(), tail), ("./", "")) 

397 

398 uri = ButlerURI(".", forceAbsolute=False) 

399 head, tail = uri.split() 

400 self.assertEqual((head.geturl(), tail), ("./", "")) 

401 

402 

403if __name__ == "__main__": 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true

404 unittest.main()