Coverage for tests/test_cliCmdQueryDatasets.py: 47%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

49 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from numpy import array 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

38 

39 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

40 storageClassFactory = StorageClassFactory() 

41 

42 @staticmethod 

43 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

44 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

45 

46 def setUp(self): 

47 self.root = makeTestTempDir(TESTDIR) 

48 self.testRepo = MetricTestRepo(self.root, configFile=self.configFile) 

49 

50 def tearDown(self): 

51 removeTestTempDir(self.root) 

52 

53 def testShowURI(self): 

54 """Test for expected output with show_uri=True.""" 

55 tables = self._queryDatasets(repo=self.root, show_uri=True) 

56 

57 expectedTables = ( 

58 AstropyTable( 

59 array( 

60 ( 

61 ( 

62 "test_metric_comp.data", 

63 "ingest/run", 

64 "R", 

65 "DummyCamComp", 

66 "d-r", 

67 "1", 

68 "423", 

69 self.testRepo.butler.datastore.root.join( 

70 "ingest/run/test_metric_comp.data/" 

71 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

72 ), 

73 ), 

74 ( 

75 "test_metric_comp.data", 

76 "ingest/run", 

77 "R", 

78 "DummyCamComp", 

79 "d-r", 

80 "1", 

81 "424", 

82 self.testRepo.butler.datastore.root.join( 

83 "ingest/run/test_metric_comp.data/" 

84 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

85 ), 

86 ), 

87 ) 

88 ), 

89 names=( 

90 "type", 

91 "run", 

92 "band", 

93 "instrument", 

94 "physical_filter", 

95 "visit_system", 

96 "visit", 

97 "URI", 

98 ), 

99 ), 

100 AstropyTable( 

101 array( 

102 ( 

103 ( 

104 "test_metric_comp.output", 

105 "ingest/run", 

106 "R", 

107 "DummyCamComp", 

108 "d-r", 

109 "1", 

110 "423", 

111 self.testRepo.butler.datastore.root.join( 

112 "ingest/run/test_metric_comp.output/" 

113 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

114 ), 

115 ), 

116 ( 

117 "test_metric_comp.output", 

118 "ingest/run", 

119 "R", 

120 "DummyCamComp", 

121 "d-r", 

122 "1", 

123 "424", 

124 self.testRepo.butler.datastore.root.join( 

125 "ingest/run/test_metric_comp.output/" 

126 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

127 ), 

128 ), 

129 ) 

130 ), 

131 names=( 

132 "type", 

133 "run", 

134 "band", 

135 "instrument", 

136 "physical_filter", 

137 "visit_system", 

138 "visit", 

139 "URI", 

140 ), 

141 ), 

142 AstropyTable( 

143 array( 

144 ( 

145 ( 

146 "test_metric_comp.summary", 

147 "ingest/run", 

148 "R", 

149 "DummyCamComp", 

150 "d-r", 

151 "1", 

152 "423", 

153 self.testRepo.butler.datastore.root.join( 

154 "ingest/run/test_metric_comp.summary/" 

155 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

156 ), 

157 ), 

158 ( 

159 "test_metric_comp.summary", 

160 "ingest/run", 

161 "R", 

162 "DummyCamComp", 

163 "d-r", 

164 "1", 

165 "424", 

166 self.testRepo.butler.datastore.root.join( 

167 "ingest/run/test_metric_comp.summary/" 

168 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

169 ), 

170 ), 

171 ) 

172 ), 

173 names=( 

174 "type", 

175 "run", 

176 "band", 

177 "instrument", 

178 "physical_filter", 

179 "visit_system", 

180 "visit", 

181 "URI", 

182 ), 

183 ), 

184 ) 

185 

186 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

187 

188 def testNoShowURI(self): 

189 """Test for expected output without show_uri (default is False).""" 

190 tables = self._queryDatasets(repo=self.root) 

191 

192 expectedTables = ( 

193 AstropyTable( 

194 array( 

195 ( 

196 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "423"), 

197 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "424"), 

198 ) 

199 ), 

200 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"), 

201 ), 

202 ) 

203 

204 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

205 

206 def testWhere(self): 

207 """Test using the where clause to reduce the number of rows returned by 

208 queryDatasets. 

209 """ 

210 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") 

211 

212 expectedTables = ( 

213 AstropyTable( 

214 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "423")), 

215 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"), 

216 ), 

217 ) 

218 

219 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

220 

221 def testGlobDatasetType(self): 

222 """Test specifying dataset type.""" 

223 # Create and register an additional DatasetType 

224 

225 self.testRepo.butler.registry.insertDimensionData( 

226 "visit", 

227 { 

228 "instrument": "DummyCamComp", 

229 "id": 425, 

230 "name": "fourtwentyfive", 

231 "physical_filter": "d-r", 

232 "visit_system": 1, 

233 }, 

234 ) 

235 

236 datasetType = addDatasetType( 

237 self.testRepo.butler, 

238 "alt_test_metric_comp", 

239 ("instrument", "visit"), 

240 "StructuredCompositeReadComp", 

241 ) 

242 

243 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

244 

245 # verify the new dataset type increases the number of tables found: 

246 tables = self._queryDatasets(repo=self.root) 

247 

248 expectedTables = ( 

249 AstropyTable( 

250 array( 

251 ( 

252 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "423"), 

253 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "424"), 

254 ) 

255 ), 

256 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"), 

257 ), 

258 AstropyTable( 

259 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "425"))), 

260 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"), 

261 ), 

262 ) 

263 

264 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

265 

266 def testFindFirstAndCollections(self): 

267 """Test the find-first option, and the collections option, since it 

268 is required for find-first.""" 

269 

270 # Add a new run, and add a dataset to shadow an existing dataset. 

271 self.testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

272 

273 # Verify that without find-first, duplicate datasets are returned 

274 tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True) 

275 

276 expectedTables = ( 

277 AstropyTable( 

278 array( 

279 ( 

280 ( 

281 "test_metric_comp.data", 

282 "foo", 

283 "3", 

284 "R", 

285 "DummyCamComp", 

286 "d-r", 

287 "1", 

288 "424", 

289 self.testRepo.butler.datastore.root.join( 

290 "foo/test_metric_comp.data/" 

291 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

292 ), 

293 ), 

294 ( 

295 "test_metric_comp.data", 

296 "ingest/run", 

297 "1", 

298 "R", 

299 "DummyCamComp", 

300 "d-r", 

301 "1", 

302 "423", 

303 self.testRepo.butler.datastore.root.join( 

304 "ingest/run/test_metric_comp.data/" 

305 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

306 ), 

307 ), 

308 ( 

309 "test_metric_comp.data", 

310 "ingest/run", 

311 "2", 

312 "R", 

313 "DummyCamComp", 

314 "d-r", 

315 "1", 

316 "424", 

317 self.testRepo.butler.datastore.root.join( 

318 "ingest/run/test_metric_comp.data/" 

319 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

320 ), 

321 ), 

322 ) 

323 ), 

324 names=( 

325 "type", 

326 "run", 

327 "id", 

328 "band", 

329 "instrument", 

330 "physical_filter", 

331 "visit_system", 

332 "visit", 

333 "URI", 

334 ), 

335 ), 

336 AstropyTable( 

337 array( 

338 ( 

339 ( 

340 "test_metric_comp.output", 

341 "foo", 

342 "3", 

343 "R", 

344 "DummyCamComp", 

345 "d-r", 

346 "1", 

347 "424", 

348 self.testRepo.butler.datastore.root.join( 

349 "foo/test_metric_comp.output/" 

350 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

351 ), 

352 ), 

353 ( 

354 "test_metric_comp.output", 

355 "ingest/run", 

356 "1", 

357 "R", 

358 "DummyCamComp", 

359 "d-r", 

360 "1", 

361 "423", 

362 self.testRepo.butler.datastore.root.join( 

363 "ingest/run/test_metric_comp.output/" 

364 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

365 ), 

366 ), 

367 ( 

368 "test_metric_comp.output", 

369 "ingest/run", 

370 "2", 

371 "R", 

372 "DummyCamComp", 

373 "d-r", 

374 "1", 

375 "424", 

376 self.testRepo.butler.datastore.root.join( 

377 "ingest/run/test_metric_comp.output/" 

378 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

379 ), 

380 ), 

381 ) 

382 ), 

383 names=( 

384 "type", 

385 "run", 

386 "id", 

387 "band", 

388 "instrument", 

389 "physical_filter", 

390 "visit_system", 

391 "visit", 

392 "URI", 

393 ), 

394 ), 

395 AstropyTable( 

396 array( 

397 ( 

398 ( 

399 "test_metric_comp.summary", 

400 "foo", 

401 "3", 

402 "R", 

403 "DummyCamComp", 

404 "d-r", 

405 "1", 

406 "424", 

407 self.testRepo.butler.datastore.root.join( 

408 "foo/test_metric_comp.summary/" 

409 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

410 ), 

411 ), 

412 ( 

413 "test_metric_comp.summary", 

414 "ingest/run", 

415 "1", 

416 "R", 

417 "DummyCamComp", 

418 "d-r", 

419 "1", 

420 "423", 

421 self.testRepo.butler.datastore.root.join( 

422 "ingest/run/test_metric_comp.summary/" 

423 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

424 ), 

425 ), 

426 ( 

427 "test_metric_comp.summary", 

428 "ingest/run", 

429 "2", 

430 "R", 

431 "DummyCamComp", 

432 "d-r", 

433 "1", 

434 "424", 

435 self.testRepo.butler.datastore.root.join( 

436 "ingest/run/test_metric_comp.summary/" 

437 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

438 ), 

439 ), 

440 ) 

441 ), 

442 names=( 

443 "type", 

444 "run", 

445 "id", 

446 "band", 

447 "instrument", 

448 "physical_filter", 

449 "visit_system", 

450 "visit", 

451 "URI", 

452 ), 

453 ), 

454 ) 

455 

456 self.assertAstropyTablesEqual(tables, expectedTables) 

457 

458 # Verify that with find first the duplicate dataset is eliminated and 

459 # the more recent dataset is returned. 

460 tables = self._queryDatasets( 

461 repo=self.root, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

462 ) 

463 

464 expectedTables = ( 

465 AstropyTable( 

466 array( 

467 ( 

468 ( 

469 "test_metric_comp.data", 

470 "foo", 

471 "3", 

472 "R", 

473 "DummyCamComp", 

474 "d-r", 

475 "1", 

476 "424", 

477 self.testRepo.butler.datastore.root.join( 

478 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

479 ), 

480 ), 

481 ( 

482 "test_metric_comp.data", 

483 "ingest/run", 

484 "1", 

485 "R", 

486 "DummyCamComp", 

487 "d-r", 

488 "1", 

489 "423", 

490 self.testRepo.butler.datastore.root.join( 

491 "ingest/run/test_metric_comp.data/" 

492 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

493 ), 

494 ), 

495 ) 

496 ), 

497 names=( 

498 "type", 

499 "run", 

500 "id", 

501 "band", 

502 "instrument", 

503 "physical_filter", 

504 "visit_system", 

505 "visit", 

506 "URI", 

507 ), 

508 ), 

509 AstropyTable( 

510 array( 

511 ( 

512 ( 

513 "test_metric_comp.output", 

514 "foo", 

515 "3", 

516 "R", 

517 "DummyCamComp", 

518 "d-r", 

519 "1", 

520 "424", 

521 self.testRepo.butler.datastore.root.join( 

522 "foo/test_metric_comp.output/" 

523 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

524 ), 

525 ), 

526 ( 

527 "test_metric_comp.output", 

528 "ingest/run", 

529 "1", 

530 "R", 

531 "DummyCamComp", 

532 "d-r", 

533 "1", 

534 "423", 

535 self.testRepo.butler.datastore.root.join( 

536 "ingest/run/test_metric_comp.output/" 

537 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

538 ), 

539 ), 

540 ) 

541 ), 

542 names=( 

543 "type", 

544 "run", 

545 "id", 

546 "band", 

547 "instrument", 

548 "physical_filter", 

549 "visit_system", 

550 "visit", 

551 "URI", 

552 ), 

553 ), 

554 AstropyTable( 

555 array( 

556 ( 

557 ( 

558 "test_metric_comp.summary", 

559 "foo", 

560 "3", 

561 "R", 

562 "DummyCamComp", 

563 "d-r", 

564 "1", 

565 "424", 

566 self.testRepo.butler.datastore.root.join( 

567 "foo/test_metric_comp.summary/" 

568 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

569 ), 

570 ), 

571 ( 

572 "test_metric_comp.summary", 

573 "ingest/run", 

574 "1", 

575 "R", 

576 "DummyCamComp", 

577 "d-r", 

578 "1", 

579 "423", 

580 self.testRepo.butler.datastore.root.join( 

581 "ingest/run/test_metric_comp.summary/" 

582 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

583 ), 

584 ), 

585 ) 

586 ), 

587 names=( 

588 "type", 

589 "run", 

590 "id", 

591 "band", 

592 "instrument", 

593 "physical_filter", 

594 "visit_system", 

595 "visit", 

596 "URI", 

597 ), 

598 ), 

599 ) 

600 

601 self.assertAstropyTablesEqual(tables, expectedTables) 

602 

603 

604if __name__ == "__main__": 604 ↛ 605line 604 didn't jump to line 605, because the condition on line 604 was never true

605 unittest.main()