Coverage for tests/test_cliCmdQueryDatasets.py: 41%

58 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-28 10:10 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from lsst.resources import ResourcePath 

33from numpy import array 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def expectedFilesystemDatastoreTables(root: ResourcePath): 

39 """Return the expected table contents.""" 

40 return ( 

41 AstropyTable( 

42 array( 

43 ( 

44 ( 

45 "test_metric_comp.data", 

46 "ingest/run", 

47 "R", 

48 "DummyCamComp", 

49 "d-r", 

50 "423", 

51 root.join( 

52 "ingest/run/test_metric_comp.data/" 

53 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

54 ), 

55 ), 

56 ( 

57 "test_metric_comp.data", 

58 "ingest/run", 

59 "R", 

60 "DummyCamComp", 

61 "d-r", 

62 "424", 

63 root.join( 

64 "ingest/run/test_metric_comp.data/" 

65 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

66 ), 

67 ), 

68 ) 

69 ), 

70 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

71 ), 

72 AstropyTable( 

73 array( 

74 ( 

75 ( 

76 "test_metric_comp.output", 

77 "ingest/run", 

78 "R", 

79 "DummyCamComp", 

80 "d-r", 

81 "423", 

82 root.join( 

83 "ingest/run/test_metric_comp.output/" 

84 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

85 ), 

86 ), 

87 ( 

88 "test_metric_comp.output", 

89 "ingest/run", 

90 "R", 

91 "DummyCamComp", 

92 "d-r", 

93 "424", 

94 root.join( 

95 "ingest/run/test_metric_comp.output/" 

96 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

97 ), 

98 ), 

99 ) 

100 ), 

101 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

102 ), 

103 AstropyTable( 

104 array( 

105 ( 

106 ( 

107 "test_metric_comp.summary", 

108 "ingest/run", 

109 "R", 

110 "DummyCamComp", 

111 "d-r", 

112 "423", 

113 root.join( 

114 "ingest/run/test_metric_comp.summary/" 

115 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

116 ), 

117 ), 

118 ( 

119 "test_metric_comp.summary", 

120 "ingest/run", 

121 "R", 

122 "DummyCamComp", 

123 "d-r", 

124 "424", 

125 root.join( 

126 "ingest/run/test_metric_comp.summary/" 

127 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

128 ), 

129 ), 

130 ) 

131 ), 

132 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

133 ), 

134 ) 

135 

136 

137class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

138 """Test the query-datasets command-line.""" 

139 

140 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

141 storageClassFactory = StorageClassFactory() 

142 

143 @staticmethod 

144 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

145 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

146 

147 def setUp(self): 

148 self.testdir = makeTestTempDir(TESTDIR) 

149 self.repoDir = os.path.join(self.testdir, "repo") 

150 

151 def tearDown(self): 

152 removeTestTempDir(self.testdir) 

153 

154 def testChained(self): 

155 testRepo = MetricTestRepo( 

156 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") 

157 ) 

158 

159 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

160 

161 self.assertAstropyTablesEqual( 

162 tables, 

163 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root), 

164 filterColumns=True, 

165 ) 

166 

167 def testShowURI(self): 

168 """Test for expected output with show_uri=True.""" 

169 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

170 

171 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

172 

173 self.assertAstropyTablesEqual( 

174 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True 

175 ) 

176 

177 def testNoShowURI(self): 

178 """Test for expected output without show_uri (default is False).""" 

179 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

180 

181 tables = self._queryDatasets(repo=self.repoDir) 

182 

183 expectedTables = ( 

184 AstropyTable( 

185 array( 

186 ( 

187 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

188 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

189 ) 

190 ), 

191 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

192 ), 

193 ) 

194 

195 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

196 

197 def testWhere(self): 

198 """Test using the where clause to reduce the number of rows returned by 

199 queryDatasets. 

200 """ 

201 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

202 

203 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

204 

205 expectedTables = ( 

206 AstropyTable( 

207 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

208 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

209 ), 

210 ) 

211 

212 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

213 

214 def testGlobDatasetType(self): 

215 """Test specifying dataset type.""" 

216 # Create and register an additional DatasetType 

217 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

218 

219 testRepo.butler.registry.insertDimensionData( 

220 "visit", 

221 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

222 ) 

223 

224 datasetType = addDatasetType( 

225 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

226 ) 

227 

228 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

229 

230 # verify the new dataset type increases the number of tables found: 

231 tables = self._queryDatasets(repo=self.repoDir) 

232 

233 expectedTables = ( 

234 AstropyTable( 

235 array( 

236 ( 

237 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

238 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

239 ) 

240 ), 

241 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

242 ), 

243 AstropyTable( 

244 array(("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425")), 

245 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

246 ), 

247 ) 

248 

249 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

250 

251 def testFindFirstAndCollections(self): 

252 """Test the find-first option, and the collections option, since it 

253 is required for find-first. 

254 """ 

255 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

256 

257 # Add a new run, and add a dataset to shadow an existing dataset. 

258 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

259 

260 # Verify that without find-first, duplicate datasets are returned 

261 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

262 

263 expectedTables = ( 

264 AstropyTable( 

265 array( 

266 ( 

267 ( 

268 "test_metric_comp.data", 

269 "foo", 

270 "R", 

271 "DummyCamComp", 

272 "d-r", 

273 "424", 

274 testRepo.butler.datastore.root.join( 

275 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

276 ), 

277 ), 

278 ( 

279 "test_metric_comp.data", 

280 "ingest/run", 

281 "R", 

282 "DummyCamComp", 

283 "d-r", 

284 "423", 

285 testRepo.butler.datastore.root.join( 

286 "ingest/run/test_metric_comp.data/" 

287 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

288 ), 

289 ), 

290 ( 

291 "test_metric_comp.data", 

292 "ingest/run", 

293 "R", 

294 "DummyCamComp", 

295 "d-r", 

296 "424", 

297 testRepo.butler.datastore.root.join( 

298 "ingest/run/test_metric_comp.data/" 

299 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

300 ), 

301 ), 

302 ) 

303 ), 

304 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

305 ), 

306 AstropyTable( 

307 array( 

308 ( 

309 ( 

310 "test_metric_comp.output", 

311 "foo", 

312 "R", 

313 "DummyCamComp", 

314 "d-r", 

315 "424", 

316 testRepo.butler.datastore.root.join( 

317 "foo/test_metric_comp.output/" 

318 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

319 ), 

320 ), 

321 ( 

322 "test_metric_comp.output", 

323 "ingest/run", 

324 "R", 

325 "DummyCamComp", 

326 "d-r", 

327 "423", 

328 testRepo.butler.datastore.root.join( 

329 "ingest/run/test_metric_comp.output/" 

330 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

331 ), 

332 ), 

333 ( 

334 "test_metric_comp.output", 

335 "ingest/run", 

336 "R", 

337 "DummyCamComp", 

338 "d-r", 

339 "424", 

340 testRepo.butler.datastore.root.join( 

341 "ingest/run/test_metric_comp.output/" 

342 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

343 ), 

344 ), 

345 ) 

346 ), 

347 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

348 ), 

349 AstropyTable( 

350 array( 

351 ( 

352 ( 

353 "test_metric_comp.summary", 

354 "foo", 

355 "R", 

356 "DummyCamComp", 

357 "d-r", 

358 "424", 

359 testRepo.butler.datastore.root.join( 

360 "foo/test_metric_comp.summary/" 

361 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

362 ), 

363 ), 

364 ( 

365 "test_metric_comp.summary", 

366 "ingest/run", 

367 "R", 

368 "DummyCamComp", 

369 "d-r", 

370 "423", 

371 testRepo.butler.datastore.root.join( 

372 "ingest/run/test_metric_comp.summary/" 

373 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

374 ), 

375 ), 

376 ( 

377 "test_metric_comp.summary", 

378 "ingest/run", 

379 "R", 

380 "DummyCamComp", 

381 "d-r", 

382 "424", 

383 testRepo.butler.datastore.root.join( 

384 "ingest/run/test_metric_comp.summary/" 

385 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

386 ), 

387 ), 

388 ) 

389 ), 

390 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

391 ), 

392 ) 

393 

394 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

395 

396 # Verify that with find first the duplicate dataset is eliminated and 

397 # the more recent dataset is returned. 

398 tables = self._queryDatasets( 

399 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

400 ) 

401 

402 expectedTables = ( 

403 AstropyTable( 

404 array( 

405 ( 

406 ( 

407 "test_metric_comp.data", 

408 "foo", 

409 "R", 

410 "DummyCamComp", 

411 "d-r", 

412 "424", 

413 testRepo.butler.datastore.root.join( 

414 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

415 ), 

416 ), 

417 ( 

418 "test_metric_comp.data", 

419 "ingest/run", 

420 "R", 

421 "DummyCamComp", 

422 "d-r", 

423 "423", 

424 testRepo.butler.datastore.root.join( 

425 "ingest/run/test_metric_comp.data/" 

426 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

427 ), 

428 ), 

429 ) 

430 ), 

431 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

432 ), 

433 AstropyTable( 

434 array( 

435 ( 

436 ( 

437 "test_metric_comp.output", 

438 "foo", 

439 "R", 

440 "DummyCamComp", 

441 "d-r", 

442 "424", 

443 testRepo.butler.datastore.root.join( 

444 "foo/test_metric_comp.output/" 

445 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

446 ), 

447 ), 

448 ( 

449 "test_metric_comp.output", 

450 "ingest/run", 

451 "R", 

452 "DummyCamComp", 

453 "d-r", 

454 "423", 

455 testRepo.butler.datastore.root.join( 

456 "ingest/run/test_metric_comp.output/" 

457 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

458 ), 

459 ), 

460 ) 

461 ), 

462 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

463 ), 

464 AstropyTable( 

465 array( 

466 ( 

467 ( 

468 "test_metric_comp.summary", 

469 "foo", 

470 "R", 

471 "DummyCamComp", 

472 "d-r", 

473 "424", 

474 testRepo.butler.datastore.root.join( 

475 "foo/test_metric_comp.summary/" 

476 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

477 ), 

478 ), 

479 ( 

480 "test_metric_comp.summary", 

481 "ingest/run", 

482 "R", 

483 "DummyCamComp", 

484 "d-r", 

485 "423", 

486 testRepo.butler.datastore.root.join( 

487 "ingest/run/test_metric_comp.summary/" 

488 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

489 ), 

490 ), 

491 ) 

492 ), 

493 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

494 ), 

495 ) 

496 

497 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

498 

499 

500if __name__ == "__main__": 

501 unittest.main()