Coverage for tests/test_cliCmdQueryDatasets.py: 42%

60 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-17 02:01 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from lsst.resources import ResourcePath 

33from numpy import array 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def expectedFilesystemDatastoreTables(root: ResourcePath): 

39 return ( 

40 AstropyTable( 

41 array( 

42 ( 

43 ( 

44 "test_metric_comp.data", 

45 "ingest/run", 

46 "R", 

47 "DummyCamComp", 

48 "d-r", 

49 "423", 

50 root.join( 

51 "ingest/run/test_metric_comp.data/" 

52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

53 ), 

54 ), 

55 ( 

56 "test_metric_comp.data", 

57 "ingest/run", 

58 "R", 

59 "DummyCamComp", 

60 "d-r", 

61 "424", 

62 root.join( 

63 "ingest/run/test_metric_comp.data/" 

64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

65 ), 

66 ), 

67 ) 

68 ), 

69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

70 ), 

71 AstropyTable( 

72 array( 

73 ( 

74 ( 

75 "test_metric_comp.output", 

76 "ingest/run", 

77 "R", 

78 "DummyCamComp", 

79 "d-r", 

80 "423", 

81 root.join( 

82 "ingest/run/test_metric_comp.output/" 

83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

84 ), 

85 ), 

86 ( 

87 "test_metric_comp.output", 

88 "ingest/run", 

89 "R", 

90 "DummyCamComp", 

91 "d-r", 

92 "424", 

93 root.join( 

94 "ingest/run/test_metric_comp.output/" 

95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

96 ), 

97 ), 

98 ) 

99 ), 

100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

101 ), 

102 AstropyTable( 

103 array( 

104 ( 

105 ( 

106 "test_metric_comp.summary", 

107 "ingest/run", 

108 "R", 

109 "DummyCamComp", 

110 "d-r", 

111 "423", 

112 root.join( 

113 "ingest/run/test_metric_comp.summary/" 

114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

115 ), 

116 ), 

117 ( 

118 "test_metric_comp.summary", 

119 "ingest/run", 

120 "R", 

121 "DummyCamComp", 

122 "d-r", 

123 "424", 

124 root.join( 

125 "ingest/run/test_metric_comp.summary/" 

126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

127 ), 

128 ), 

129 ) 

130 ), 

131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

132 ), 

133 ) 

134 

135 

136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

137 

138 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

139 storageClassFactory = StorageClassFactory() 

140 

141 @staticmethod 

142 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

143 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

144 

145 def setUp(self): 

146 self.testdir = makeTestTempDir(TESTDIR) 

147 self.repoDir = os.path.join(self.testdir, "repo") 

148 

149 def tearDown(self): 

150 removeTestTempDir(self.testdir) 

151 

152 def testChained(self): 

153 testRepo = MetricTestRepo( 

154 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") 

155 ) 

156 

157 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

158 

159 self.assertAstropyTablesEqual( 

160 tables, 

161 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root), 

162 filterColumns=True, 

163 ) 

164 

165 def testShowURI(self): 

166 """Test for expected output with show_uri=True.""" 

167 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

168 

169 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

170 

171 self.assertAstropyTablesEqual( 

172 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True 

173 ) 

174 

175 def testNoShowURI(self): 

176 """Test for expected output without show_uri (default is False).""" 

177 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

178 

179 tables = self._queryDatasets(repo=self.repoDir) 

180 

181 expectedTables = ( 

182 AstropyTable( 

183 array( 

184 ( 

185 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

186 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

187 ) 

188 ), 

189 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

190 ), 

191 ) 

192 

193 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

194 

195 def testWhere(self): 

196 """Test using the where clause to reduce the number of rows returned by 

197 queryDatasets. 

198 """ 

199 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

200 

201 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

202 

203 expectedTables = ( 

204 AstropyTable( 

205 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

206 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

207 ), 

208 ) 

209 

210 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

211 

212 def testGlobDatasetType(self): 

213 """Test specifying dataset type.""" 

214 # Create and register an additional DatasetType 

215 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

216 

217 testRepo.butler.registry.insertDimensionData( 

218 "visit", 

219 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

220 ) 

221 

222 datasetType = addDatasetType( 

223 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

224 ) 

225 

226 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

227 

228 # verify the new dataset type increases the number of tables found: 

229 tables = self._queryDatasets(repo=self.repoDir) 

230 

231 expectedTables = ( 

232 AstropyTable( 

233 array( 

234 ( 

235 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

236 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

237 ) 

238 ), 

239 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

240 ), 

241 AstropyTable( 

242 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))), 

243 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

244 ), 

245 ) 

246 

247 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

248 

249 def testFindFirstAndCollections(self): 

250 """Test the find-first option, and the collections option, since it 

251 is required for find-first.""" 

252 

253 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

254 

255 # Add a new run, and add a dataset to shadow an existing dataset. 

256 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

257 

258 # Verify that without find-first, duplicate datasets are returned 

259 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

260 

261 expectedTables = ( 

262 AstropyTable( 

263 array( 

264 ( 

265 ( 

266 "test_metric_comp.data", 

267 "foo", 

268 "R", 

269 "DummyCamComp", 

270 "d-r", 

271 "424", 

272 testRepo.butler.datastore.root.join( 

273 "foo/test_metric_comp.data/" 

274 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

275 ), 

276 ), 

277 ( 

278 "test_metric_comp.data", 

279 "ingest/run", 

280 "R", 

281 "DummyCamComp", 

282 "d-r", 

283 "423", 

284 testRepo.butler.datastore.root.join( 

285 "ingest/run/test_metric_comp.data/" 

286 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

287 ), 

288 ), 

289 ( 

290 "test_metric_comp.data", 

291 "ingest/run", 

292 "R", 

293 "DummyCamComp", 

294 "d-r", 

295 "424", 

296 testRepo.butler.datastore.root.join( 

297 "ingest/run/test_metric_comp.data/" 

298 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

299 ), 

300 ), 

301 ) 

302 ), 

303 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

304 ), 

305 AstropyTable( 

306 array( 

307 ( 

308 ( 

309 "test_metric_comp.output", 

310 "foo", 

311 "R", 

312 "DummyCamComp", 

313 "d-r", 

314 "424", 

315 testRepo.butler.datastore.root.join( 

316 "foo/test_metric_comp.output/" 

317 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

318 ), 

319 ), 

320 ( 

321 "test_metric_comp.output", 

322 "ingest/run", 

323 "R", 

324 "DummyCamComp", 

325 "d-r", 

326 "423", 

327 testRepo.butler.datastore.root.join( 

328 "ingest/run/test_metric_comp.output/" 

329 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

330 ), 

331 ), 

332 ( 

333 "test_metric_comp.output", 

334 "ingest/run", 

335 "R", 

336 "DummyCamComp", 

337 "d-r", 

338 "424", 

339 testRepo.butler.datastore.root.join( 

340 "ingest/run/test_metric_comp.output/" 

341 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

342 ), 

343 ), 

344 ) 

345 ), 

346 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

347 ), 

348 AstropyTable( 

349 array( 

350 ( 

351 ( 

352 "test_metric_comp.summary", 

353 "foo", 

354 "R", 

355 "DummyCamComp", 

356 "d-r", 

357 "424", 

358 testRepo.butler.datastore.root.join( 

359 "foo/test_metric_comp.summary/" 

360 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

361 ), 

362 ), 

363 ( 

364 "test_metric_comp.summary", 

365 "ingest/run", 

366 "R", 

367 "DummyCamComp", 

368 "d-r", 

369 "423", 

370 testRepo.butler.datastore.root.join( 

371 "ingest/run/test_metric_comp.summary/" 

372 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

373 ), 

374 ), 

375 ( 

376 "test_metric_comp.summary", 

377 "ingest/run", 

378 "R", 

379 "DummyCamComp", 

380 "d-r", 

381 "424", 

382 testRepo.butler.datastore.root.join( 

383 "ingest/run/test_metric_comp.summary/" 

384 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

385 ), 

386 ), 

387 ) 

388 ), 

389 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

390 ), 

391 ) 

392 

393 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

394 

395 # Verify that with find first the duplicate dataset is eliminated and 

396 # the more recent dataset is returned. 

397 tables = self._queryDatasets( 

398 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

399 ) 

400 

401 expectedTables = ( 

402 AstropyTable( 

403 array( 

404 ( 

405 ( 

406 "test_metric_comp.data", 

407 "foo", 

408 "R", 

409 "DummyCamComp", 

410 "d-r", 

411 "424", 

412 testRepo.butler.datastore.root.join( 

413 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

414 ), 

415 ), 

416 ( 

417 "test_metric_comp.data", 

418 "ingest/run", 

419 "R", 

420 "DummyCamComp", 

421 "d-r", 

422 "423", 

423 testRepo.butler.datastore.root.join( 

424 "ingest/run/test_metric_comp.data/" 

425 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

426 ), 

427 ), 

428 ) 

429 ), 

430 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

431 ), 

432 AstropyTable( 

433 array( 

434 ( 

435 ( 

436 "test_metric_comp.output", 

437 "foo", 

438 "R", 

439 "DummyCamComp", 

440 "d-r", 

441 "424", 

442 testRepo.butler.datastore.root.join( 

443 "foo/test_metric_comp.output/" 

444 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

445 ), 

446 ), 

447 ( 

448 "test_metric_comp.output", 

449 "ingest/run", 

450 "R", 

451 "DummyCamComp", 

452 "d-r", 

453 "423", 

454 testRepo.butler.datastore.root.join( 

455 "ingest/run/test_metric_comp.output/" 

456 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

457 ), 

458 ), 

459 ) 

460 ), 

461 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

462 ), 

463 AstropyTable( 

464 array( 

465 ( 

466 ( 

467 "test_metric_comp.summary", 

468 "foo", 

469 "R", 

470 "DummyCamComp", 

471 "d-r", 

472 "424", 

473 testRepo.butler.datastore.root.join( 

474 "foo/test_metric_comp.summary/" 

475 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

476 ), 

477 ), 

478 ( 

479 "test_metric_comp.summary", 

480 "ingest/run", 

481 "R", 

482 "DummyCamComp", 

483 "d-r", 

484 "423", 

485 testRepo.butler.datastore.root.join( 

486 "ingest/run/test_metric_comp.summary/" 

487 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

488 ), 

489 ), 

490 ) 

491 ), 

492 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

493 ), 

494 ) 

495 

496 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

497 

498 

499if __name__ == "__main__": 499 ↛ 500line 499 didn't jump to line 500, because the condition on line 499 was never true

500 unittest.main()