Coverage for tests/test_cliCmdQueryDatasets.py: 39%

65 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-26 02:48 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Unit tests for daf_butler CLI query-datasets command. 

29""" 

30 

31import os 

32import unittest 

33 

34from astropy.table import Table as AstropyTable 

35from lsst.daf.butler import StorageClassFactory, script 

36from lsst.daf.butler.tests import addDatasetType 

37from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

38from lsst.resources import ResourcePath 

39from numpy import array 

40 

41TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

42 

43 

44def expectedFilesystemDatastoreTables(root: ResourcePath): 

45 """Return the expected table contents.""" 

46 return ( 

47 AstropyTable( 

48 array( 

49 ( 

50 ( 

51 "test_metric_comp.data", 

52 "ingest/run", 

53 "DummyCamComp", 

54 "423", 

55 "R", 

56 "d-r", 

57 root.join( 

58 "ingest/run/test_metric_comp.data/" 

59 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

60 ), 

61 ), 

62 ( 

63 "test_metric_comp.data", 

64 "ingest/run", 

65 "DummyCamComp", 

66 "424", 

67 "R", 

68 "d-r", 

69 root.join( 

70 "ingest/run/test_metric_comp.data/" 

71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

72 ), 

73 ), 

74 ) 

75 ), 

76 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

77 ), 

78 AstropyTable( 

79 array( 

80 ( 

81 ( 

82 "test_metric_comp.output", 

83 "ingest/run", 

84 "DummyCamComp", 

85 "423", 

86 "R", 

87 "d-r", 

88 root.join( 

89 "ingest/run/test_metric_comp.output/" 

90 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

91 ), 

92 ), 

93 ( 

94 "test_metric_comp.output", 

95 "ingest/run", 

96 "DummyCamComp", 

97 "424", 

98 "R", 

99 "d-r", 

100 root.join( 

101 "ingest/run/test_metric_comp.output/" 

102 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

103 ), 

104 ), 

105 ) 

106 ), 

107 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

108 ), 

109 AstropyTable( 

110 array( 

111 ( 

112 ( 

113 "test_metric_comp.summary", 

114 "ingest/run", 

115 "DummyCamComp", 

116 "423", 

117 "R", 

118 "d-r", 

119 root.join( 

120 "ingest/run/test_metric_comp.summary/" 

121 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

122 ), 

123 ), 

124 ( 

125 "test_metric_comp.summary", 

126 "ingest/run", 

127 "DummyCamComp", 

128 "424", 

129 "R", 

130 "d-r", 

131 root.join( 

132 "ingest/run/test_metric_comp.summary/" 

133 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

134 ), 

135 ), 

136 ) 

137 ), 

138 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

139 ), 

140 ) 

141 

142 

143class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

144 """Test the query-datasets command-line.""" 

145 

146 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

147 storageClassFactory = StorageClassFactory() 

148 

149 @staticmethod 

150 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

151 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

152 

153 def setUp(self): 

154 self.testdir = makeTestTempDir(TESTDIR) 

155 self.repoDir = os.path.join(self.testdir, "repo") 

156 

157 def tearDown(self): 

158 removeTestTempDir(self.testdir) 

159 

160 def testChained(self): 

161 testRepo = MetricTestRepo( 

162 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") 

163 ) 

164 

165 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

166 

167 # Want second datastore root. 

168 roots = testRepo.butler.get_datastore_roots() 

169 datastore_root = roots[testRepo.butler.get_datastore_names()[1]] 

170 

171 self.assertAstropyTablesEqual( 

172 tables, 

173 expectedFilesystemDatastoreTables(datastore_root), 

174 filterColumns=True, 

175 ) 

176 

177 def testShowURI(self): 

178 """Test for expected output with show_uri=True.""" 

179 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

180 

181 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

182 

183 roots = testRepo.butler.get_datastore_roots() 

184 datastore_root = list(roots.values())[0] 

185 

186 self.assertAstropyTablesEqual( 

187 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True 

188 ) 

189 

190 def testNoShowURI(self): 

191 """Test for expected output without show_uri (default is False).""" 

192 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

193 

194 tables = self._queryDatasets(repo=self.repoDir) 

195 

196 expectedTables = ( 

197 AstropyTable( 

198 array( 

199 ( 

200 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"), 

201 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"), 

202 ) 

203 ), 

204 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

205 ), 

206 ) 

207 

208 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

209 

210 def testWhere(self): 

211 """Test using the where clause to reduce the number of rows returned by 

212 queryDatasets. 

213 """ 

214 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

215 

216 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

217 

218 expectedTables = ( 

219 AstropyTable( 

220 array(("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r")), 

221 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

222 ), 

223 ) 

224 

225 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

226 

227 def testGlobDatasetType(self): 

228 """Test specifying dataset type.""" 

229 # Create and register an additional DatasetType 

230 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

231 

232 testRepo.butler.registry.insertDimensionData( 

233 "visit", 

234 { 

235 "instrument": "DummyCamComp", 

236 "id": 425, 

237 "name": "fourtwentyfive", 

238 "physical_filter": "d-r", 

239 "day_obs": 20200101, 

240 }, 

241 ) 

242 

243 datasetType = addDatasetType( 

244 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

245 ) 

246 

247 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

248 

249 # verify the new dataset type increases the number of tables found: 

250 tables = self._queryDatasets(repo=self.repoDir) 

251 

252 expectedTables = ( 

253 AstropyTable( 

254 array( 

255 ( 

256 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"), 

257 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"), 

258 ) 

259 ), 

260 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

261 ), 

262 AstropyTable( 

263 array(("alt_test_metric_comp", "ingest/run", "DummyCamComp", "425", "R", "d-r")), 

264 names=("type", "run", "instrument", "visit", "band", "physical_filter"), 

265 ), 

266 ) 

267 

268 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

269 

270 def testFindFirstAndCollections(self): 

271 """Test the find-first option, and the collections option, since it 

272 is required for find-first. 

273 """ 

274 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

275 

276 # Add a new run, and add a dataset to shadow an existing dataset. 

277 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

278 

279 # Verify that without find-first, duplicate datasets are returned 

280 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

281 

282 # The test should be running with a single FileDatastore. 

283 roots = testRepo.butler.get_datastore_roots() 

284 assert len(roots) == 1 

285 datastore_root = list(roots.values())[0] 

286 

287 expectedTables = ( 

288 AstropyTable( 

289 array( 

290 ( 

291 ( 

292 "test_metric_comp.data", 

293 "foo", 

294 "DummyCamComp", 

295 "424", 

296 "R", 

297 "d-r", 

298 datastore_root.join( 

299 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

300 ), 

301 ), 

302 ( 

303 "test_metric_comp.data", 

304 "ingest/run", 

305 "DummyCamComp", 

306 "423", 

307 "R", 

308 "d-r", 

309 datastore_root.join( 

310 "ingest/run/test_metric_comp.data/" 

311 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

312 ), 

313 ), 

314 ( 

315 "test_metric_comp.data", 

316 "ingest/run", 

317 "DummyCamComp", 

318 "424", 

319 "R", 

320 "d-r", 

321 datastore_root.join( 

322 "ingest/run/test_metric_comp.data/" 

323 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

324 ), 

325 ), 

326 ) 

327 ), 

328 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

329 ), 

330 AstropyTable( 

331 array( 

332 ( 

333 ( 

334 "test_metric_comp.output", 

335 "foo", 

336 "DummyCamComp", 

337 "424", 

338 "R", 

339 "d-r", 

340 datastore_root.join( 

341 "foo/test_metric_comp.output/" 

342 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

343 ), 

344 ), 

345 ( 

346 "test_metric_comp.output", 

347 "ingest/run", 

348 "DummyCamComp", 

349 "423", 

350 "R", 

351 "d-r", 

352 datastore_root.join( 

353 "ingest/run/test_metric_comp.output/" 

354 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

355 ), 

356 ), 

357 ( 

358 "test_metric_comp.output", 

359 "ingest/run", 

360 "DummyCamComp", 

361 "424", 

362 "R", 

363 "d-r", 

364 datastore_root.join( 

365 "ingest/run/test_metric_comp.output/" 

366 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

367 ), 

368 ), 

369 ) 

370 ), 

371 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

372 ), 

373 AstropyTable( 

374 array( 

375 ( 

376 ( 

377 "test_metric_comp.summary", 

378 "foo", 

379 "DummyCamComp", 

380 "424", 

381 "R", 

382 "d-r", 

383 datastore_root.join( 

384 "foo/test_metric_comp.summary/" 

385 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

386 ), 

387 ), 

388 ( 

389 "test_metric_comp.summary", 

390 "ingest/run", 

391 "DummyCamComp", 

392 "423", 

393 "R", 

394 "d-r", 

395 datastore_root.join( 

396 "ingest/run/test_metric_comp.summary/" 

397 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

398 ), 

399 ), 

400 ( 

401 "test_metric_comp.summary", 

402 "ingest/run", 

403 "DummyCamComp", 

404 "424", 

405 "R", 

406 "d-r", 

407 datastore_root.join( 

408 "ingest/run/test_metric_comp.summary/" 

409 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

410 ), 

411 ), 

412 ) 

413 ), 

414 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

415 ), 

416 ) 

417 

418 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

419 

420 # Verify that with find first the duplicate dataset is eliminated and 

421 # the more recent dataset is returned. 

422 tables = self._queryDatasets( 

423 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

424 ) 

425 

426 expectedTables = ( 

427 AstropyTable( 

428 array( 

429 ( 

430 ( 

431 "test_metric_comp.data", 

432 "foo", 

433 "DummyCamComp", 

434 "424", 

435 "R", 

436 "d-r", 

437 datastore_root.join( 

438 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

439 ), 

440 ), 

441 ( 

442 "test_metric_comp.data", 

443 "ingest/run", 

444 "DummyCamComp", 

445 "423", 

446 "R", 

447 "d-r", 

448 datastore_root.join( 

449 "ingest/run/test_metric_comp.data/" 

450 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

451 ), 

452 ), 

453 ) 

454 ), 

455 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

456 ), 

457 AstropyTable( 

458 array( 

459 ( 

460 ( 

461 "test_metric_comp.output", 

462 "foo", 

463 "DummyCamComp", 

464 "424", 

465 "R", 

466 "d-r", 

467 datastore_root.join( 

468 "foo/test_metric_comp.output/" 

469 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

470 ), 

471 ), 

472 ( 

473 "test_metric_comp.output", 

474 "ingest/run", 

475 "DummyCamComp", 

476 "423", 

477 "R", 

478 "d-r", 

479 datastore_root.join( 

480 "ingest/run/test_metric_comp.output/" 

481 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

482 ), 

483 ), 

484 ) 

485 ), 

486 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

487 ), 

488 AstropyTable( 

489 array( 

490 ( 

491 ( 

492 "test_metric_comp.summary", 

493 "foo", 

494 "DummyCamComp", 

495 "424", 

496 "R", 

497 "d-r", 

498 datastore_root.join( 

499 "foo/test_metric_comp.summary/" 

500 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

501 ), 

502 ), 

503 ( 

504 "test_metric_comp.summary", 

505 "ingest/run", 

506 "DummyCamComp", 

507 "423", 

508 "R", 

509 "d-r", 

510 datastore_root.join( 

511 "ingest/run/test_metric_comp.summary/" 

512 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

513 ), 

514 ), 

515 ) 

516 ), 

517 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"), 

518 ), 

519 ) 

520 

521 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

522 

523 

524if __name__ == "__main__": 

525 unittest.main()