Coverage for tests/test_cliCmdQueryDatasets.py: 39%

65 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-16 10:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for daf_butler CLI query-datasets command. 

29""" 

30 

31import os 

32import unittest 

33 

34from astropy.table import Table as AstropyTable 

35from lsst.daf.butler import StorageClassFactory, script 

36from lsst.daf.butler.tests import addDatasetType 

37from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

38from lsst.resources import ResourcePath 

39from numpy import array 

40 

41TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

42 

43 

44def expectedFilesystemDatastoreTables(root: ResourcePath): 

45 """Return the expected table contents.""" 

46 return ( 

47 AstropyTable( 

48 array( 

49 ( 

50 ( 

51 "test_metric_comp.data", 

52 "ingest/run", 

53 "DummyCamComp", 

54 "423", 

55 "R", 

56 "d-r", 

57 root.join( 

58 "ingest/run/test_metric_comp.data/" 

59 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

60 ), 

61 ), 

62 ( 

63 "test_metric_comp.data", 

64 "ingest/run", 

65 "DummyCamComp", 

66 "424", 

67 "R", 

68 "d-r", 

69 root.join( 

70 "ingest/run/test_metric_comp.data/" 

71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

72 ), 

73 ), 

74 ) 

75 ), 

76 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

77 ), 

78 AstropyTable( 

79 array( 

80 ( 

81 ( 

82 "test_metric_comp.output", 

83 "ingest/run", 

84 "DummyCamComp", 

85 "423", 

86 "R", 

87 "d-r", 

88 root.join( 

89 "ingest/run/test_metric_comp.output/" 

90 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

91 ), 

92 ), 

93 ( 

94 "test_metric_comp.output", 

95 "ingest/run", 

96 "DummyCamComp", 

97 "424", 

98 "R", 

99 "d-r", 

100 root.join( 

101 "ingest/run/test_metric_comp.output/" 

102 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

103 ), 

104 ), 

105 ) 

106 ), 

107 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

108 ), 

109 AstropyTable( 

110 array( 

111 ( 

112 ( 

113 "test_metric_comp.summary", 

114 "ingest/run", 

115 "DummyCamComp", 

116 "423", 

117 "R", 

118 "d-r", 

119 root.join( 

120 "ingest/run/test_metric_comp.summary/" 

121 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

122 ), 

123 ), 

124 ( 

125 "test_metric_comp.summary", 

126 "ingest/run", 

127 "DummyCamComp", 

128 "424", 

129 "R", 

130 "d-r", 

131 root.join( 

132 "ingest/run/test_metric_comp.summary/" 

133 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

134 ), 

135 ), 

136 ) 

137 ), 

138 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

139 ), 

140 ) 

141 

142 

143class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

144 """Test the query-datasets command-line.""" 

145 

146 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

147 storageClassFactory = StorageClassFactory() 

148 

149 @staticmethod 

150 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

151 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

152 

153 def setUp(self): 

154 self.testdir = makeTestTempDir(TESTDIR) 

155 self.repoDir = os.path.join(self.testdir, "repo") 

156 

157 def tearDown(self): 

158 removeTestTempDir(self.testdir) 

159 

160 def testChained(self): 

161 testRepo = MetricTestRepo( 

162 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") 

163 ) 

164 

165 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

166 

167 # Want second datastore root. 

168 roots = testRepo.butler.get_datastore_roots() 

169 datastore_root = roots[testRepo.butler.get_datastore_names()[1]] 

170 

171 self.assertAstropyTablesEqual( 

172 tables, 

173 expectedFilesystemDatastoreTables(datastore_root), 

174 filterColumns=True, 

175 ) 

176 

177 def testShowURI(self): 

178 """Test for expected output with show_uri=True.""" 

179 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

180 

181 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

182 

183 roots = testRepo.butler.get_datastore_roots() 

184 datastore_root = list(roots.values())[0] 

185 

186 self.assertAstropyTablesEqual( 

187 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True 

188 ) 

189 

190 def testNoShowURI(self): 

191 """Test for expected output without show_uri (default is False).""" 

192 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

193 

194 tables = self._queryDatasets(repo=self.repoDir) 

195 

196 expectedTables = ( 

197 AstropyTable( 

198 array( 

199 ( 

200 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"), 

201 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"), 

202 ) 

203 ), 

204 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

205 ), 

206 ) 

207 

208 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

209 

210 def testWhere(self): 

211 """Test using the where clause to reduce the number of rows returned by 

212 queryDatasets. 

213 """ 

214 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

215 

216 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

217 

218 expectedTables = ( 

219 AstropyTable( 

220 array(("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r")), 

221 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

222 ), 

223 ) 

224 

225 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

226 

227 def testGlobDatasetType(self): 

228 """Test specifying dataset type.""" 

229 # Create and register an additional DatasetType 

230 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

231 

232 testRepo.butler.registry.insertDimensionData( 

233 "visit", 

234 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

235 ) 

236 

237 datasetType = addDatasetType( 

238 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

239 ) 

240 

241 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

242 

243 # verify the new dataset type increases the number of tables found: 

244 tables = self._queryDatasets(repo=self.repoDir) 

245 

246 expectedTables = ( 

247 AstropyTable( 

248 array( 

249 ( 

250 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"), 

251 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"), 

252 ) 

253 ), 

254 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

255 ), 

256 AstropyTable( 

257 array(("alt_test_metric_comp", "ingest/run", "DummyCamComp", "425", "R", "d-r")), 

258 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

259 ), 

260 ) 

261 

262 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

263 

264 def testFindFirstAndCollections(self): 

265 """Test the find-first option, and the collections option, since it 

266 is required for find-first. 

267 """ 

268 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

269 

270 # Add a new run, and add a dataset to shadow an existing dataset. 

271 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

272 

273 # Verify that without find-first, duplicate datasets are returned 

274 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

275 

276 # The test should be running with a single FileDatastore. 

277 roots = testRepo.butler.get_datastore_roots() 

278 assert len(roots) == 1 

279 datastore_root = list(roots.values())[0] 

280 

281 expectedTables = ( 

282 AstropyTable( 

283 array( 

284 ( 

285 ( 

286 "test_metric_comp.data", 

287 "foo", 

288 "DummyCamComp", 

289 "424", 

290 "R", 

291 "d-r", 

292 datastore_root.join( 

293 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

294 ), 

295 ), 

296 ( 

297 "test_metric_comp.data", 

298 "ingest/run", 

299 "DummyCamComp", 

300 "423", 

301 "R", 

302 "d-r", 

303 datastore_root.join( 

304 "ingest/run/test_metric_comp.data/" 

305 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

306 ), 

307 ), 

308 ( 

309 "test_metric_comp.data", 

310 "ingest/run", 

311 "DummyCamComp", 

312 "424", 

313 "R", 

314 "d-r", 

315 datastore_root.join( 

316 "ingest/run/test_metric_comp.data/" 

317 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

318 ), 

319 ), 

320 ) 

321 ), 

322 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

323 ), 

324 AstropyTable( 

325 array( 

326 ( 

327 ( 

328 "test_metric_comp.output", 

329 "foo", 

330 "DummyCamComp", 

331 "424", 

332 "R", 

333 "d-r", 

334 datastore_root.join( 

335 "foo/test_metric_comp.output/" 

336 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

337 ), 

338 ), 

339 ( 

340 "test_metric_comp.output", 

341 "ingest/run", 

342 "DummyCamComp", 

343 "423", 

344 "R", 

345 "d-r", 

346 datastore_root.join( 

347 "ingest/run/test_metric_comp.output/" 

348 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

349 ), 

350 ), 

351 ( 

352 "test_metric_comp.output", 

353 "ingest/run", 

354 "DummyCamComp", 

355 "424", 

356 "R", 

357 "d-r", 

358 datastore_root.join( 

359 "ingest/run/test_metric_comp.output/" 

360 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

361 ), 

362 ), 

363 ) 

364 ), 

365 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

366 ), 

367 AstropyTable( 

368 array( 

369 ( 

370 ( 

371 "test_metric_comp.summary", 

372 "foo", 

373 "DummyCamComp", 

374 "424", 

375 "R", 

376 "d-r", 

377 datastore_root.join( 

378 "foo/test_metric_comp.summary/" 

379 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

380 ), 

381 ), 

382 ( 

383 "test_metric_comp.summary", 

384 "ingest/run", 

385 "DummyCamComp", 

386 "423", 

387 "R", 

388 "d-r", 

389 datastore_root.join( 

390 "ingest/run/test_metric_comp.summary/" 

391 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

392 ), 

393 ), 

394 ( 

395 "test_metric_comp.summary", 

396 "ingest/run", 

397 "DummyCamComp", 

398 "424", 

399 "R", 

400 "d-r", 

401 datastore_root.join( 

402 "ingest/run/test_metric_comp.summary/" 

403 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

404 ), 

405 ), 

406 ) 

407 ), 

408 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

409 ), 

410 ) 

411 

412 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

413 

414 # Verify that with find first the duplicate dataset is eliminated and 

415 # the more recent dataset is returned. 

416 tables = self._queryDatasets( 

417 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

418 ) 

419 

420 expectedTables = ( 

421 AstropyTable( 

422 array( 

423 ( 

424 ( 

425 "test_metric_comp.data", 

426 "foo", 

427 "DummyCamComp", 

428 "424", 

429 "R", 

430 "d-r", 

431 datastore_root.join( 

432 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

433 ), 

434 ), 

435 ( 

436 "test_metric_comp.data", 

437 "ingest/run", 

438 "DummyCamComp", 

439 "423", 

440 "R", 

441 "d-r", 

442 datastore_root.join( 

443 "ingest/run/test_metric_comp.data/" 

444 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

445 ), 

446 ), 

447 ) 

448 ), 

449 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

450 ), 

451 AstropyTable( 

452 array( 

453 ( 

454 ( 

455 "test_metric_comp.output", 

456 "foo", 

457 "DummyCamComp", 

458 "424", 

459 "R", 

460 "d-r", 

461 datastore_root.join( 

462 "foo/test_metric_comp.output/" 

463 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

464 ), 

465 ), 

466 ( 

467 "test_metric_comp.output", 

468 "ingest/run", 

469 "DummyCamComp", 

470 "423", 

471 "R", 

472 "d-r", 

473 datastore_root.join( 

474 "ingest/run/test_metric_comp.output/" 

475 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

476 ), 

477 ), 

478 ) 

479 ), 

480 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

481 ), 

482 AstropyTable( 

483 array( 

484 ( 

485 ( 

486 "test_metric_comp.summary", 

487 "foo", 

488 "DummyCamComp", 

489 "424", 

490 "R", 

491 "d-r", 

492 datastore_root.join( 

493 "foo/test_metric_comp.summary/" 

494 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

495 ), 

496 ), 

497 ( 

498 "test_metric_comp.summary", 

499 "ingest/run", 

500 "DummyCamComp", 

501 "423", 

502 "R", 

503 "d-r", 

504 datastore_root.join( 

505 "ingest/run/test_metric_comp.summary/" 

506 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

507 ), 

508 ), 

509 ) 

510 ), 

511 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

512 ), 

513 ) 

514 

515 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

516 

517 

518if __name__ == "__main__": 

519 unittest.main()