Coverage for tests/test_cliCmdQueryDatasets.py: 47%

49 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-09 02:25 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from numpy import array 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

38 

39 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

40 storageClassFactory = StorageClassFactory() 

41 

42 @staticmethod 

43 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

44 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

45 

46 def setUp(self): 

47 self.root = makeTestTempDir(TESTDIR) 

48 self.testRepo = MetricTestRepo(self.root, configFile=self.configFile) 

49 

50 def tearDown(self): 

51 removeTestTempDir(self.root) 

52 

53 def testShowURI(self): 

54 """Test for expected output with show_uri=True.""" 

55 tables = self._queryDatasets(repo=self.root, show_uri=True) 

56 

57 expectedTables = ( 

58 AstropyTable( 

59 array( 

60 ( 

61 ( 

62 "test_metric_comp.data", 

63 "ingest/run", 

64 "R", 

65 "DummyCamComp", 

66 "d-r", 

67 "423", 

68 self.testRepo.butler.datastore.root.join( 

69 "ingest/run/test_metric_comp.data/" 

70 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

71 ), 

72 ), 

73 ( 

74 "test_metric_comp.data", 

75 "ingest/run", 

76 "R", 

77 "DummyCamComp", 

78 "d-r", 

79 "424", 

80 self.testRepo.butler.datastore.root.join( 

81 "ingest/run/test_metric_comp.data/" 

82 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

83 ), 

84 ), 

85 ) 

86 ), 

87 names=( 

88 "type", 

89 "run", 

90 "band", 

91 "instrument", 

92 "physical_filter", 

93 "visit", 

94 "URI", 

95 ), 

96 ), 

97 AstropyTable( 

98 array( 

99 ( 

100 ( 

101 "test_metric_comp.output", 

102 "ingest/run", 

103 "R", 

104 "DummyCamComp", 

105 "d-r", 

106 "423", 

107 self.testRepo.butler.datastore.root.join( 

108 "ingest/run/test_metric_comp.output/" 

109 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

110 ), 

111 ), 

112 ( 

113 "test_metric_comp.output", 

114 "ingest/run", 

115 "R", 

116 "DummyCamComp", 

117 "d-r", 

118 "424", 

119 self.testRepo.butler.datastore.root.join( 

120 "ingest/run/test_metric_comp.output/" 

121 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

122 ), 

123 ), 

124 ) 

125 ), 

126 names=( 

127 "type", 

128 "run", 

129 "band", 

130 "instrument", 

131 "physical_filter", 

132 "visit", 

133 "URI", 

134 ), 

135 ), 

136 AstropyTable( 

137 array( 

138 ( 

139 ( 

140 "test_metric_comp.summary", 

141 "ingest/run", 

142 "R", 

143 "DummyCamComp", 

144 "d-r", 

145 "423", 

146 self.testRepo.butler.datastore.root.join( 

147 "ingest/run/test_metric_comp.summary/" 

148 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

149 ), 

150 ), 

151 ( 

152 "test_metric_comp.summary", 

153 "ingest/run", 

154 "R", 

155 "DummyCamComp", 

156 "d-r", 

157 "424", 

158 self.testRepo.butler.datastore.root.join( 

159 "ingest/run/test_metric_comp.summary/" 

160 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

161 ), 

162 ), 

163 ) 

164 ), 

165 names=( 

166 "type", 

167 "run", 

168 "band", 

169 "instrument", 

170 "physical_filter", 

171 "visit", 

172 "URI", 

173 ), 

174 ), 

175 ) 

176 

177 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

178 

179 def testNoShowURI(self): 

180 """Test for expected output without show_uri (default is False).""" 

181 tables = self._queryDatasets(repo=self.root) 

182 

183 expectedTables = ( 

184 AstropyTable( 

185 array( 

186 ( 

187 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

188 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

189 ) 

190 ), 

191 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

192 ), 

193 ) 

194 

195 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

196 

197 def testWhere(self): 

198 """Test using the where clause to reduce the number of rows returned by 

199 queryDatasets. 

200 """ 

201 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") 

202 

203 expectedTables = ( 

204 AstropyTable( 

205 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

206 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

207 ), 

208 ) 

209 

210 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

211 

212 def testGlobDatasetType(self): 

213 """Test specifying dataset type.""" 

214 # Create and register an additional DatasetType 

215 

216 self.testRepo.butler.registry.insertDimensionData( 

217 "visit", 

218 { 

219 "instrument": "DummyCamComp", 

220 "id": 425, 

221 "name": "fourtwentyfive", 

222 "physical_filter": "d-r", 

223 }, 

224 ) 

225 

226 datasetType = addDatasetType( 

227 self.testRepo.butler, 

228 "alt_test_metric_comp", 

229 ("instrument", "visit"), 

230 "StructuredCompositeReadComp", 

231 ) 

232 

233 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

234 

235 # verify the new dataset type increases the number of tables found: 

236 tables = self._queryDatasets(repo=self.root) 

237 

238 expectedTables = ( 

239 AstropyTable( 

240 array( 

241 ( 

242 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

243 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

244 ) 

245 ), 

246 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

247 ), 

248 AstropyTable( 

249 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))), 

250 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

251 ), 

252 ) 

253 

254 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

255 

256 def testFindFirstAndCollections(self): 

257 """Test the find-first option, and the collections option, since it 

258 is required for find-first.""" 

259 

260 # Add a new run, and add a dataset to shadow an existing dataset. 

261 self.testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

262 

263 # Verify that without find-first, duplicate datasets are returned 

264 tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True) 

265 

266 expectedTables = ( 

267 AstropyTable( 

268 array( 

269 ( 

270 ( 

271 "test_metric_comp.data", 

272 "foo", 

273 "3", 

274 "R", 

275 "DummyCamComp", 

276 "d-r", 

277 "424", 

278 self.testRepo.butler.datastore.root.join( 

279 "foo/test_metric_comp.data/" 

280 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

281 ), 

282 ), 

283 ( 

284 "test_metric_comp.data", 

285 "ingest/run", 

286 "1", 

287 "R", 

288 "DummyCamComp", 

289 "d-r", 

290 "423", 

291 self.testRepo.butler.datastore.root.join( 

292 "ingest/run/test_metric_comp.data/" 

293 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

294 ), 

295 ), 

296 ( 

297 "test_metric_comp.data", 

298 "ingest/run", 

299 "2", 

300 "R", 

301 "DummyCamComp", 

302 "d-r", 

303 "424", 

304 self.testRepo.butler.datastore.root.join( 

305 "ingest/run/test_metric_comp.data/" 

306 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

307 ), 

308 ), 

309 ) 

310 ), 

311 names=( 

312 "type", 

313 "run", 

314 "id", 

315 "band", 

316 "instrument", 

317 "physical_filter", 

318 "visit", 

319 "URI", 

320 ), 

321 ), 

322 AstropyTable( 

323 array( 

324 ( 

325 ( 

326 "test_metric_comp.output", 

327 "foo", 

328 "3", 

329 "R", 

330 "DummyCamComp", 

331 "d-r", 

332 "424", 

333 self.testRepo.butler.datastore.root.join( 

334 "foo/test_metric_comp.output/" 

335 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

336 ), 

337 ), 

338 ( 

339 "test_metric_comp.output", 

340 "ingest/run", 

341 "1", 

342 "R", 

343 "DummyCamComp", 

344 "d-r", 

345 "423", 

346 self.testRepo.butler.datastore.root.join( 

347 "ingest/run/test_metric_comp.output/" 

348 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

349 ), 

350 ), 

351 ( 

352 "test_metric_comp.output", 

353 "ingest/run", 

354 "2", 

355 "R", 

356 "DummyCamComp", 

357 "d-r", 

358 "424", 

359 self.testRepo.butler.datastore.root.join( 

360 "ingest/run/test_metric_comp.output/" 

361 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

362 ), 

363 ), 

364 ) 

365 ), 

366 names=( 

367 "type", 

368 "run", 

369 "id", 

370 "band", 

371 "instrument", 

372 "physical_filter", 

373 "visit", 

374 "URI", 

375 ), 

376 ), 

377 AstropyTable( 

378 array( 

379 ( 

380 ( 

381 "test_metric_comp.summary", 

382 "foo", 

383 "3", 

384 "R", 

385 "DummyCamComp", 

386 "d-r", 

387 "424", 

388 self.testRepo.butler.datastore.root.join( 

389 "foo/test_metric_comp.summary/" 

390 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

391 ), 

392 ), 

393 ( 

394 "test_metric_comp.summary", 

395 "ingest/run", 

396 "1", 

397 "R", 

398 "DummyCamComp", 

399 "d-r", 

400 "423", 

401 self.testRepo.butler.datastore.root.join( 

402 "ingest/run/test_metric_comp.summary/" 

403 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

404 ), 

405 ), 

406 ( 

407 "test_metric_comp.summary", 

408 "ingest/run", 

409 "2", 

410 "R", 

411 "DummyCamComp", 

412 "d-r", 

413 "424", 

414 self.testRepo.butler.datastore.root.join( 

415 "ingest/run/test_metric_comp.summary/" 

416 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

417 ), 

418 ), 

419 ) 

420 ), 

421 names=( 

422 "type", 

423 "run", 

424 "id", 

425 "band", 

426 "instrument", 

427 "physical_filter", 

428 "visit", 

429 "URI", 

430 ), 

431 ), 

432 ) 

433 

434 self.assertAstropyTablesEqual(tables, expectedTables) 

435 

436 # Verify that with find first the duplicate dataset is eliminated and 

437 # the more recent dataset is returned. 

438 tables = self._queryDatasets( 

439 repo=self.root, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

440 ) 

441 

442 expectedTables = ( 

443 AstropyTable( 

444 array( 

445 ( 

446 ( 

447 "test_metric_comp.data", 

448 "foo", 

449 "3", 

450 "R", 

451 "DummyCamComp", 

452 "d-r", 

453 "424", 

454 self.testRepo.butler.datastore.root.join( 

455 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

456 ), 

457 ), 

458 ( 

459 "test_metric_comp.data", 

460 "ingest/run", 

461 "1", 

462 "R", 

463 "DummyCamComp", 

464 "d-r", 

465 "423", 

466 self.testRepo.butler.datastore.root.join( 

467 "ingest/run/test_metric_comp.data/" 

468 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

469 ), 

470 ), 

471 ) 

472 ), 

473 names=( 

474 "type", 

475 "run", 

476 "id", 

477 "band", 

478 "instrument", 

479 "physical_filter", 

480 "visit", 

481 "URI", 

482 ), 

483 ), 

484 AstropyTable( 

485 array( 

486 ( 

487 ( 

488 "test_metric_comp.output", 

489 "foo", 

490 "3", 

491 "R", 

492 "DummyCamComp", 

493 "d-r", 

494 "424", 

495 self.testRepo.butler.datastore.root.join( 

496 "foo/test_metric_comp.output/" 

497 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

498 ), 

499 ), 

500 ( 

501 "test_metric_comp.output", 

502 "ingest/run", 

503 "1", 

504 "R", 

505 "DummyCamComp", 

506 "d-r", 

507 "423", 

508 self.testRepo.butler.datastore.root.join( 

509 "ingest/run/test_metric_comp.output/" 

510 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

511 ), 

512 ), 

513 ) 

514 ), 

515 names=( 

516 "type", 

517 "run", 

518 "id", 

519 "band", 

520 "instrument", 

521 "physical_filter", 

522 "visit", 

523 "URI", 

524 ), 

525 ), 

526 AstropyTable( 

527 array( 

528 ( 

529 ( 

530 "test_metric_comp.summary", 

531 "foo", 

532 "3", 

533 "R", 

534 "DummyCamComp", 

535 "d-r", 

536 "424", 

537 self.testRepo.butler.datastore.root.join( 

538 "foo/test_metric_comp.summary/" 

539 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

540 ), 

541 ), 

542 ( 

543 "test_metric_comp.summary", 

544 "ingest/run", 

545 "1", 

546 "R", 

547 "DummyCamComp", 

548 "d-r", 

549 "423", 

550 self.testRepo.butler.datastore.root.join( 

551 "ingest/run/test_metric_comp.summary/" 

552 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

553 ), 

554 ), 

555 ) 

556 ), 

557 names=( 

558 "type", 

559 "run", 

560 "id", 

561 "band", 

562 "instrument", 

563 "physical_filter", 

564 "visit", 

565 "URI", 

566 ), 

567 ), 

568 ) 

569 

570 self.assertAstropyTablesEqual(tables, expectedTables) 

571 

572 

573if __name__ == "__main__": 573 ↛ 574line 573 didn't jump to line 574, because the condition on line 573 was never true

574 unittest.main()