Coverage for tests/test_cliCmdQueryDatasets.py: 37%

65 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 10:56 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from lsst.resources import ResourcePath 

33from numpy import array 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def expectedFilesystemDatastoreTables(root: ResourcePath): 

39 """Return the expected table contents.""" 

40 return ( 

41 AstropyTable( 

42 array( 

43 ( 

44 ( 

45 "test_metric_comp.data", 

46 "ingest/run", 

47 "R", 

48 "DummyCamComp", 

49 "d-r", 

50 "423", 

51 root.join( 

52 "ingest/run/test_metric_comp.data/" 

53 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

54 ), 

55 ), 

56 ( 

57 "test_metric_comp.data", 

58 "ingest/run", 

59 "R", 

60 "DummyCamComp", 

61 "d-r", 

62 "424", 

63 root.join( 

64 "ingest/run/test_metric_comp.data/" 

65 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

66 ), 

67 ), 

68 ) 

69 ), 

70 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

71 ), 

72 AstropyTable( 

73 array( 

74 ( 

75 ( 

76 "test_metric_comp.output", 

77 "ingest/run", 

78 "R", 

79 "DummyCamComp", 

80 "d-r", 

81 "423", 

82 root.join( 

83 "ingest/run/test_metric_comp.output/" 

84 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

85 ), 

86 ), 

87 ( 

88 "test_metric_comp.output", 

89 "ingest/run", 

90 "R", 

91 "DummyCamComp", 

92 "d-r", 

93 "424", 

94 root.join( 

95 "ingest/run/test_metric_comp.output/" 

96 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

97 ), 

98 ), 

99 ) 

100 ), 

101 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

102 ), 

103 AstropyTable( 

104 array( 

105 ( 

106 ( 

107 "test_metric_comp.summary", 

108 "ingest/run", 

109 "R", 

110 "DummyCamComp", 

111 "d-r", 

112 "423", 

113 root.join( 

114 "ingest/run/test_metric_comp.summary/" 

115 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

116 ), 

117 ), 

118 ( 

119 "test_metric_comp.summary", 

120 "ingest/run", 

121 "R", 

122 "DummyCamComp", 

123 "d-r", 

124 "424", 

125 root.join( 

126 "ingest/run/test_metric_comp.summary/" 

127 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

128 ), 

129 ), 

130 ) 

131 ), 

132 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

133 ), 

134 ) 

135 

136 

137class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

138 """Test the query-datasets command-line.""" 

139 

140 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

141 storageClassFactory = StorageClassFactory() 

142 

143 @staticmethod 

144 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

145 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

146 

147 def setUp(self): 

148 self.testdir = makeTestTempDir(TESTDIR) 

149 self.repoDir = os.path.join(self.testdir, "repo") 

150 

151 def tearDown(self): 

152 removeTestTempDir(self.testdir) 

153 

154 def testChained(self): 

155 testRepo = MetricTestRepo( 

156 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") 

157 ) 

158 

159 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

160 

161 # Want second datastore root. 

162 roots = testRepo.butler.get_datastore_roots() 

163 datastore_root = roots[testRepo.butler.get_datastore_names()[1]] 

164 

165 self.assertAstropyTablesEqual( 

166 tables, 

167 expectedFilesystemDatastoreTables(datastore_root), 

168 filterColumns=True, 

169 ) 

170 

171 def testShowURI(self): 

172 """Test for expected output with show_uri=True.""" 

173 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

174 

175 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

176 

177 roots = testRepo.butler.get_datastore_roots() 

178 datastore_root = list(roots.values())[0] 

179 

180 self.assertAstropyTablesEqual( 

181 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True 

182 ) 

183 

184 def testNoShowURI(self): 

185 """Test for expected output without show_uri (default is False).""" 

186 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

187 

188 tables = self._queryDatasets(repo=self.repoDir) 

189 

190 expectedTables = ( 

191 AstropyTable( 

192 array( 

193 ( 

194 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

195 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

196 ) 

197 ), 

198 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

199 ), 

200 ) 

201 

202 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

203 

204 def testWhere(self): 

205 """Test using the where clause to reduce the number of rows returned by 

206 queryDatasets. 

207 """ 

208 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

209 

210 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

211 

212 expectedTables = ( 

213 AstropyTable( 

214 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

215 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

216 ), 

217 ) 

218 

219 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

220 

221 def testGlobDatasetType(self): 

222 """Test specifying dataset type.""" 

223 # Create and register an additional DatasetType 

224 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

225 

226 testRepo.butler.registry.insertDimensionData( 

227 "visit", 

228 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

229 ) 

230 

231 datasetType = addDatasetType( 

232 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

233 ) 

234 

235 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

236 

237 # verify the new dataset type increases the number of tables found: 

238 tables = self._queryDatasets(repo=self.repoDir) 

239 

240 expectedTables = ( 

241 AstropyTable( 

242 array( 

243 ( 

244 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

245 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

246 ) 

247 ), 

248 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

249 ), 

250 AstropyTable( 

251 array(("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425")), 

252 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

253 ), 

254 ) 

255 

256 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

257 

258 def testFindFirstAndCollections(self): 

259 """Test the find-first option, and the collections option, since it 

260 is required for find-first. 

261 """ 

262 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

263 

264 # Add a new run, and add a dataset to shadow an existing dataset. 

265 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

266 

267 # Verify that without find-first, duplicate datasets are returned 

268 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

269 

270 # The test should be running with a single FileDatastore. 

271 roots = testRepo.butler.get_datastore_roots() 

272 assert len(roots) == 1 

273 datastore_root = list(roots.values())[0] 

274 

275 expectedTables = ( 

276 AstropyTable( 

277 array( 

278 ( 

279 ( 

280 "test_metric_comp.data", 

281 "foo", 

282 "R", 

283 "DummyCamComp", 

284 "d-r", 

285 "424", 

286 datastore_root.join( 

287 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

288 ), 

289 ), 

290 ( 

291 "test_metric_comp.data", 

292 "ingest/run", 

293 "R", 

294 "DummyCamComp", 

295 "d-r", 

296 "423", 

297 datastore_root.join( 

298 "ingest/run/test_metric_comp.data/" 

299 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

300 ), 

301 ), 

302 ( 

303 "test_metric_comp.data", 

304 "ingest/run", 

305 "R", 

306 "DummyCamComp", 

307 "d-r", 

308 "424", 

309 datastore_root.join( 

310 "ingest/run/test_metric_comp.data/" 

311 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

312 ), 

313 ), 

314 ) 

315 ), 

316 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

317 ), 

318 AstropyTable( 

319 array( 

320 ( 

321 ( 

322 "test_metric_comp.output", 

323 "foo", 

324 "R", 

325 "DummyCamComp", 

326 "d-r", 

327 "424", 

328 datastore_root.join( 

329 "foo/test_metric_comp.output/" 

330 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

331 ), 

332 ), 

333 ( 

334 "test_metric_comp.output", 

335 "ingest/run", 

336 "R", 

337 "DummyCamComp", 

338 "d-r", 

339 "423", 

340 datastore_root.join( 

341 "ingest/run/test_metric_comp.output/" 

342 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

343 ), 

344 ), 

345 ( 

346 "test_metric_comp.output", 

347 "ingest/run", 

348 "R", 

349 "DummyCamComp", 

350 "d-r", 

351 "424", 

352 datastore_root.join( 

353 "ingest/run/test_metric_comp.output/" 

354 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

355 ), 

356 ), 

357 ) 

358 ), 

359 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

360 ), 

361 AstropyTable( 

362 array( 

363 ( 

364 ( 

365 "test_metric_comp.summary", 

366 "foo", 

367 "R", 

368 "DummyCamComp", 

369 "d-r", 

370 "424", 

371 datastore_root.join( 

372 "foo/test_metric_comp.summary/" 

373 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

374 ), 

375 ), 

376 ( 

377 "test_metric_comp.summary", 

378 "ingest/run", 

379 "R", 

380 "DummyCamComp", 

381 "d-r", 

382 "423", 

383 datastore_root.join( 

384 "ingest/run/test_metric_comp.summary/" 

385 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

386 ), 

387 ), 

388 ( 

389 "test_metric_comp.summary", 

390 "ingest/run", 

391 "R", 

392 "DummyCamComp", 

393 "d-r", 

394 "424", 

395 datastore_root.join( 

396 "ingest/run/test_metric_comp.summary/" 

397 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

398 ), 

399 ), 

400 ) 

401 ), 

402 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

403 ), 

404 ) 

405 

406 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

407 

408 # Verify that with find first the duplicate dataset is eliminated and 

409 # the more recent dataset is returned. 

410 tables = self._queryDatasets( 

411 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

412 ) 

413 

414 expectedTables = ( 

415 AstropyTable( 

416 array( 

417 ( 

418 ( 

419 "test_metric_comp.data", 

420 "foo", 

421 "R", 

422 "DummyCamComp", 

423 "d-r", 

424 "424", 

425 datastore_root.join( 

426 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

427 ), 

428 ), 

429 ( 

430 "test_metric_comp.data", 

431 "ingest/run", 

432 "R", 

433 "DummyCamComp", 

434 "d-r", 

435 "423", 

436 datastore_root.join( 

437 "ingest/run/test_metric_comp.data/" 

438 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

439 ), 

440 ), 

441 ) 

442 ), 

443 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

444 ), 

445 AstropyTable( 

446 array( 

447 ( 

448 ( 

449 "test_metric_comp.output", 

450 "foo", 

451 "R", 

452 "DummyCamComp", 

453 "d-r", 

454 "424", 

455 datastore_root.join( 

456 "foo/test_metric_comp.output/" 

457 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

458 ), 

459 ), 

460 ( 

461 "test_metric_comp.output", 

462 "ingest/run", 

463 "R", 

464 "DummyCamComp", 

465 "d-r", 

466 "423", 

467 datastore_root.join( 

468 "ingest/run/test_metric_comp.output/" 

469 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

470 ), 

471 ), 

472 ) 

473 ), 

474 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

475 ), 

476 AstropyTable( 

477 array( 

478 ( 

479 ( 

480 "test_metric_comp.summary", 

481 "foo", 

482 "R", 

483 "DummyCamComp", 

484 "d-r", 

485 "424", 

486 datastore_root.join( 

487 "foo/test_metric_comp.summary/" 

488 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

489 ), 

490 ), 

491 ( 

492 "test_metric_comp.summary", 

493 "ingest/run", 

494 "R", 

495 "DummyCamComp", 

496 "d-r", 

497 "423", 

498 datastore_root.join( 

499 "ingest/run/test_metric_comp.summary/" 

500 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

501 ), 

502 ), 

503 ) 

504 ), 

505 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

506 ), 

507 ) 

508 

509 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

510 

511 

512if __name__ == "__main__": 

513 unittest.main()