Coverage for tests/test_cliCmdQueryDatasets.py: 42%

60 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-04 02:04 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from lsst.resources import ResourcePath 

33from numpy import array 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def expectedFilesystemDatastoreTables(root: ResourcePath): 

39 return ( 

40 AstropyTable( 

41 array( 

42 ( 

43 ( 

44 "test_metric_comp.data", 

45 "ingest/run", 

46 "R", 

47 "DummyCamComp", 

48 "d-r", 

49 "423", 

50 root.join( 

51 "ingest/run/test_metric_comp.data/" 

52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

53 ), 

54 ), 

55 ( 

56 "test_metric_comp.data", 

57 "ingest/run", 

58 "R", 

59 "DummyCamComp", 

60 "d-r", 

61 "424", 

62 root.join( 

63 "ingest/run/test_metric_comp.data/" 

64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

65 ), 

66 ), 

67 ) 

68 ), 

69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

70 ), 

71 AstropyTable( 

72 array( 

73 ( 

74 ( 

75 "test_metric_comp.output", 

76 "ingest/run", 

77 "R", 

78 "DummyCamComp", 

79 "d-r", 

80 "423", 

81 root.join( 

82 "ingest/run/test_metric_comp.output/" 

83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

84 ), 

85 ), 

86 ( 

87 "test_metric_comp.output", 

88 "ingest/run", 

89 "R", 

90 "DummyCamComp", 

91 "d-r", 

92 "424", 

93 root.join( 

94 "ingest/run/test_metric_comp.output/" 

95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

96 ), 

97 ), 

98 ) 

99 ), 

100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

101 ), 

102 AstropyTable( 

103 array( 

104 ( 

105 ( 

106 "test_metric_comp.summary", 

107 "ingest/run", 

108 "R", 

109 "DummyCamComp", 

110 "d-r", 

111 "423", 

112 root.join( 

113 "ingest/run/test_metric_comp.summary/" 

114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

115 ), 

116 ), 

117 ( 

118 "test_metric_comp.summary", 

119 "ingest/run", 

120 "R", 

121 "DummyCamComp", 

122 "d-r", 

123 "424", 

124 root.join( 

125 "ingest/run/test_metric_comp.summary/" 

126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

127 ), 

128 ), 

129 ) 

130 ), 

131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

132 ), 

133 ) 

134 

135 

136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

137 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

138 storageClassFactory = StorageClassFactory() 

139 

140 @staticmethod 

141 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

142 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

143 

144 def setUp(self): 

145 self.testdir = makeTestTempDir(TESTDIR) 

146 self.repoDir = os.path.join(self.testdir, "repo") 

147 

148 def tearDown(self): 

149 removeTestTempDir(self.testdir) 

150 

151 def testChained(self): 

152 testRepo = MetricTestRepo( 

153 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml") 

154 ) 

155 

156 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

157 

158 self.assertAstropyTablesEqual( 

159 tables, 

160 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root), 

161 filterColumns=True, 

162 ) 

163 

164 def testShowURI(self): 

165 """Test for expected output with show_uri=True.""" 

166 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

167 

168 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

169 

170 self.assertAstropyTablesEqual( 

171 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True 

172 ) 

173 

174 def testNoShowURI(self): 

175 """Test for expected output without show_uri (default is False).""" 

176 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

177 

178 tables = self._queryDatasets(repo=self.repoDir) 

179 

180 expectedTables = ( 

181 AstropyTable( 

182 array( 

183 ( 

184 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

185 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

186 ) 

187 ), 

188 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

189 ), 

190 ) 

191 

192 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

193 

194 def testWhere(self): 

195 """Test using the where clause to reduce the number of rows returned by 

196 queryDatasets. 

197 """ 

198 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

199 

200 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

201 

202 expectedTables = ( 

203 AstropyTable( 

204 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

205 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

206 ), 

207 ) 

208 

209 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

210 

211 def testGlobDatasetType(self): 

212 """Test specifying dataset type.""" 

213 # Create and register an additional DatasetType 

214 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

215 

216 testRepo.butler.registry.insertDimensionData( 

217 "visit", 

218 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

219 ) 

220 

221 datasetType = addDatasetType( 

222 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

223 ) 

224 

225 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

226 

227 # verify the new dataset type increases the number of tables found: 

228 tables = self._queryDatasets(repo=self.repoDir) 

229 

230 expectedTables = ( 

231 AstropyTable( 

232 array( 

233 ( 

234 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

235 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

236 ) 

237 ), 

238 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

239 ), 

240 AstropyTable( 

241 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))), 

242 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

243 ), 

244 ) 

245 

246 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

247 

248 def testFindFirstAndCollections(self): 

249 """Test the find-first option, and the collections option, since it 

250 is required for find-first.""" 

251 

252 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

253 

254 # Add a new run, and add a dataset to shadow an existing dataset. 

255 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

256 

257 # Verify that without find-first, duplicate datasets are returned 

258 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

259 

260 expectedTables = ( 

261 AstropyTable( 

262 array( 

263 ( 

264 ( 

265 "test_metric_comp.data", 

266 "foo", 

267 "R", 

268 "DummyCamComp", 

269 "d-r", 

270 "424", 

271 testRepo.butler.datastore.root.join( 

272 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

273 ), 

274 ), 

275 ( 

276 "test_metric_comp.data", 

277 "ingest/run", 

278 "R", 

279 "DummyCamComp", 

280 "d-r", 

281 "423", 

282 testRepo.butler.datastore.root.join( 

283 "ingest/run/test_metric_comp.data/" 

284 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

285 ), 

286 ), 

287 ( 

288 "test_metric_comp.data", 

289 "ingest/run", 

290 "R", 

291 "DummyCamComp", 

292 "d-r", 

293 "424", 

294 testRepo.butler.datastore.root.join( 

295 "ingest/run/test_metric_comp.data/" 

296 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

297 ), 

298 ), 

299 ) 

300 ), 

301 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

302 ), 

303 AstropyTable( 

304 array( 

305 ( 

306 ( 

307 "test_metric_comp.output", 

308 "foo", 

309 "R", 

310 "DummyCamComp", 

311 "d-r", 

312 "424", 

313 testRepo.butler.datastore.root.join( 

314 "foo/test_metric_comp.output/" 

315 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

316 ), 

317 ), 

318 ( 

319 "test_metric_comp.output", 

320 "ingest/run", 

321 "R", 

322 "DummyCamComp", 

323 "d-r", 

324 "423", 

325 testRepo.butler.datastore.root.join( 

326 "ingest/run/test_metric_comp.output/" 

327 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

328 ), 

329 ), 

330 ( 

331 "test_metric_comp.output", 

332 "ingest/run", 

333 "R", 

334 "DummyCamComp", 

335 "d-r", 

336 "424", 

337 testRepo.butler.datastore.root.join( 

338 "ingest/run/test_metric_comp.output/" 

339 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

340 ), 

341 ), 

342 ) 

343 ), 

344 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

345 ), 

346 AstropyTable( 

347 array( 

348 ( 

349 ( 

350 "test_metric_comp.summary", 

351 "foo", 

352 "R", 

353 "DummyCamComp", 

354 "d-r", 

355 "424", 

356 testRepo.butler.datastore.root.join( 

357 "foo/test_metric_comp.summary/" 

358 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

359 ), 

360 ), 

361 ( 

362 "test_metric_comp.summary", 

363 "ingest/run", 

364 "R", 

365 "DummyCamComp", 

366 "d-r", 

367 "423", 

368 testRepo.butler.datastore.root.join( 

369 "ingest/run/test_metric_comp.summary/" 

370 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

371 ), 

372 ), 

373 ( 

374 "test_metric_comp.summary", 

375 "ingest/run", 

376 "R", 

377 "DummyCamComp", 

378 "d-r", 

379 "424", 

380 testRepo.butler.datastore.root.join( 

381 "ingest/run/test_metric_comp.summary/" 

382 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

383 ), 

384 ), 

385 ) 

386 ), 

387 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

388 ), 

389 ) 

390 

391 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

392 

393 # Verify that with find first the duplicate dataset is eliminated and 

394 # the more recent dataset is returned. 

395 tables = self._queryDatasets( 

396 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

397 ) 

398 

399 expectedTables = ( 

400 AstropyTable( 

401 array( 

402 ( 

403 ( 

404 "test_metric_comp.data", 

405 "foo", 

406 "R", 

407 "DummyCamComp", 

408 "d-r", 

409 "424", 

410 testRepo.butler.datastore.root.join( 

411 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

412 ), 

413 ), 

414 ( 

415 "test_metric_comp.data", 

416 "ingest/run", 

417 "R", 

418 "DummyCamComp", 

419 "d-r", 

420 "423", 

421 testRepo.butler.datastore.root.join( 

422 "ingest/run/test_metric_comp.data/" 

423 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

424 ), 

425 ), 

426 ) 

427 ), 

428 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

429 ), 

430 AstropyTable( 

431 array( 

432 ( 

433 ( 

434 "test_metric_comp.output", 

435 "foo", 

436 "R", 

437 "DummyCamComp", 

438 "d-r", 

439 "424", 

440 testRepo.butler.datastore.root.join( 

441 "foo/test_metric_comp.output/" 

442 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

443 ), 

444 ), 

445 ( 

446 "test_metric_comp.output", 

447 "ingest/run", 

448 "R", 

449 "DummyCamComp", 

450 "d-r", 

451 "423", 

452 testRepo.butler.datastore.root.join( 

453 "ingest/run/test_metric_comp.output/" 

454 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

455 ), 

456 ), 

457 ) 

458 ), 

459 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

460 ), 

461 AstropyTable( 

462 array( 

463 ( 

464 ( 

465 "test_metric_comp.summary", 

466 "foo", 

467 "R", 

468 "DummyCamComp", 

469 "d-r", 

470 "424", 

471 testRepo.butler.datastore.root.join( 

472 "foo/test_metric_comp.summary/" 

473 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

474 ), 

475 ), 

476 ( 

477 "test_metric_comp.summary", 

478 "ingest/run", 

479 "R", 

480 "DummyCamComp", 

481 "d-r", 

482 "423", 

483 testRepo.butler.datastore.root.join( 

484 "ingest/run/test_metric_comp.summary/" 

485 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

486 ), 

487 ), 

488 ) 

489 ), 

490 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

491 ), 

492 ) 

493 

494 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

495 

496 

497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true

498 unittest.main()