Coverage for tests/test_cliCmdQueryDatasets.py: 43%

61 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-15 09:41 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from lsst.resources import ResourcePath 

33from numpy import array 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def expectedFilesystemDatastoreTables(root: ResourcePath): 

39 return ( 

40 AstropyTable( 

41 array( 

42 ( 

43 ( 

44 "test_metric_comp.data", 

45 "ingest/run", 

46 "R", 

47 "DummyCamComp", 

48 "d-r", 

49 "423", 

50 root.join( 

51 "ingest/run/test_metric_comp.data/" 

52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

53 ), 

54 ), 

55 ( 

56 "test_metric_comp.data", 

57 "ingest/run", 

58 "R", 

59 "DummyCamComp", 

60 "d-r", 

61 "424", 

62 root.join( 

63 "ingest/run/test_metric_comp.data/" 

64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

65 ), 

66 ), 

67 ) 

68 ), 

69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

70 ), 

71 AstropyTable( 

72 array( 

73 ( 

74 ( 

75 "test_metric_comp.output", 

76 "ingest/run", 

77 "R", 

78 "DummyCamComp", 

79 "d-r", 

80 "423", 

81 root.join( 

82 "ingest/run/test_metric_comp.output/" 

83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

84 ), 

85 ), 

86 ( 

87 "test_metric_comp.output", 

88 "ingest/run", 

89 "R", 

90 "DummyCamComp", 

91 "d-r", 

92 "424", 

93 root.join( 

94 "ingest/run/test_metric_comp.output/" 

95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

96 ), 

97 ), 

98 ) 

99 ), 

100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

101 ), 

102 AstropyTable( 

103 array( 

104 ( 

105 ( 

106 "test_metric_comp.summary", 

107 "ingest/run", 

108 "R", 

109 "DummyCamComp", 

110 "d-r", 

111 "423", 

112 root.join( 

113 "ingest/run/test_metric_comp.summary/" 

114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

115 ), 

116 ), 

117 ( 

118 "test_metric_comp.summary", 

119 "ingest/run", 

120 "R", 

121 "DummyCamComp", 

122 "d-r", 

123 "424", 

124 root.join( 

125 "ingest/run/test_metric_comp.summary/" 

126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

127 ), 

128 ), 

129 ) 

130 ), 

131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

132 ), 

133 ) 

134 

135 

136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

137 

138 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

139 storageClassFactory = StorageClassFactory() 

140 

141 @staticmethod 

142 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

143 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

144 

145 def setUp(self): 

146 self.testdir = makeTestTempDir(TESTDIR) 

147 self.repoDir = os.path.join(self.testdir, "repo") 

148 

149 def tearDown(self): 

150 removeTestTempDir(self.testdir) 

151 

152 def testChained(self): 

153 root = makeTestTempDir(TESTDIR) 

154 testRepo = MetricTestRepo(root, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")) 

155 

156 tables = self._queryDatasets(repo=root, show_uri=True) 

157 

158 self.assertAstropyTablesEqual( 

159 tables, 

160 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root), 

161 filterColumns=True, 

162 ) 

163 

164 def testShowURI(self): 

165 """Test for expected output with show_uri=True.""" 

166 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

167 

168 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

169 

170 self.assertAstropyTablesEqual( 

171 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True 

172 ) 

173 

174 def testNoShowURI(self): 

175 """Test for expected output without show_uri (default is False).""" 

176 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

177 

178 tables = self._queryDatasets(repo=self.repoDir) 

179 

180 expectedTables = ( 

181 AstropyTable( 

182 array( 

183 ( 

184 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

185 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

186 ) 

187 ), 

188 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

189 ), 

190 ) 

191 

192 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

193 

194 def testWhere(self): 

195 """Test using the where clause to reduce the number of rows returned by 

196 queryDatasets. 

197 """ 

198 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

199 

200 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

201 

202 expectedTables = ( 

203 AstropyTable( 

204 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

205 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

206 ), 

207 ) 

208 

209 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

210 

211 def testGlobDatasetType(self): 

212 """Test specifying dataset type.""" 

213 # Create and register an additional DatasetType 

214 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

215 

216 testRepo.butler.registry.insertDimensionData( 

217 "visit", 

218 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

219 ) 

220 

221 datasetType = addDatasetType( 

222 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

223 ) 

224 

225 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

226 

227 # verify the new dataset type increases the number of tables found: 

228 tables = self._queryDatasets(repo=self.repoDir) 

229 

230 expectedTables = ( 

231 AstropyTable( 

232 array( 

233 ( 

234 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

235 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

236 ) 

237 ), 

238 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

239 ), 

240 AstropyTable( 

241 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))), 

242 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

243 ), 

244 ) 

245 

246 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

247 

248 def testFindFirstAndCollections(self): 

249 """Test the find-first option, and the collections option, since it 

250 is required for find-first.""" 

251 

252 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

253 

254 # Add a new run, and add a dataset to shadow an existing dataset. 

255 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

256 

257 # Verify that without find-first, duplicate datasets are returned 

258 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

259 

260 expectedTables = ( 

261 AstropyTable( 

262 array( 

263 ( 

264 ( 

265 "test_metric_comp.data", 

266 "foo", 

267 "R", 

268 "DummyCamComp", 

269 "d-r", 

270 "424", 

271 testRepo.butler.datastore.root.join( 

272 "foo/test_metric_comp.data/" 

273 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

274 ), 

275 ), 

276 ( 

277 "test_metric_comp.data", 

278 "ingest/run", 

279 "R", 

280 "DummyCamComp", 

281 "d-r", 

282 "423", 

283 testRepo.butler.datastore.root.join( 

284 "ingest/run/test_metric_comp.data/" 

285 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

286 ), 

287 ), 

288 ( 

289 "test_metric_comp.data", 

290 "ingest/run", 

291 "R", 

292 "DummyCamComp", 

293 "d-r", 

294 "424", 

295 testRepo.butler.datastore.root.join( 

296 "ingest/run/test_metric_comp.data/" 

297 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

298 ), 

299 ), 

300 ) 

301 ), 

302 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

303 ), 

304 AstropyTable( 

305 array( 

306 ( 

307 ( 

308 "test_metric_comp.output", 

309 "foo", 

310 "R", 

311 "DummyCamComp", 

312 "d-r", 

313 "424", 

314 testRepo.butler.datastore.root.join( 

315 "foo/test_metric_comp.output/" 

316 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

317 ), 

318 ), 

319 ( 

320 "test_metric_comp.output", 

321 "ingest/run", 

322 "R", 

323 "DummyCamComp", 

324 "d-r", 

325 "423", 

326 testRepo.butler.datastore.root.join( 

327 "ingest/run/test_metric_comp.output/" 

328 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

329 ), 

330 ), 

331 ( 

332 "test_metric_comp.output", 

333 "ingest/run", 

334 "R", 

335 "DummyCamComp", 

336 "d-r", 

337 "424", 

338 testRepo.butler.datastore.root.join( 

339 "ingest/run/test_metric_comp.output/" 

340 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

341 ), 

342 ), 

343 ) 

344 ), 

345 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

346 ), 

347 AstropyTable( 

348 array( 

349 ( 

350 ( 

351 "test_metric_comp.summary", 

352 "foo", 

353 "R", 

354 "DummyCamComp", 

355 "d-r", 

356 "424", 

357 testRepo.butler.datastore.root.join( 

358 "foo/test_metric_comp.summary/" 

359 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

360 ), 

361 ), 

362 ( 

363 "test_metric_comp.summary", 

364 "ingest/run", 

365 "R", 

366 "DummyCamComp", 

367 "d-r", 

368 "423", 

369 testRepo.butler.datastore.root.join( 

370 "ingest/run/test_metric_comp.summary/" 

371 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

372 ), 

373 ), 

374 ( 

375 "test_metric_comp.summary", 

376 "ingest/run", 

377 "R", 

378 "DummyCamComp", 

379 "d-r", 

380 "424", 

381 testRepo.butler.datastore.root.join( 

382 "ingest/run/test_metric_comp.summary/" 

383 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

384 ), 

385 ), 

386 ) 

387 ), 

388 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

389 ), 

390 ) 

391 

392 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

393 

394 # Verify that with find first the duplicate dataset is eliminated and 

395 # the more recent dataset is returned. 

396 tables = self._queryDatasets( 

397 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

398 ) 

399 

400 expectedTables = ( 

401 AstropyTable( 

402 array( 

403 ( 

404 ( 

405 "test_metric_comp.data", 

406 "foo", 

407 "R", 

408 "DummyCamComp", 

409 "d-r", 

410 "424", 

411 testRepo.butler.datastore.root.join( 

412 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

413 ), 

414 ), 

415 ( 

416 "test_metric_comp.data", 

417 "ingest/run", 

418 "R", 

419 "DummyCamComp", 

420 "d-r", 

421 "423", 

422 testRepo.butler.datastore.root.join( 

423 "ingest/run/test_metric_comp.data/" 

424 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

425 ), 

426 ), 

427 ) 

428 ), 

429 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

430 ), 

431 AstropyTable( 

432 array( 

433 ( 

434 ( 

435 "test_metric_comp.output", 

436 "foo", 

437 "R", 

438 "DummyCamComp", 

439 "d-r", 

440 "424", 

441 testRepo.butler.datastore.root.join( 

442 "foo/test_metric_comp.output/" 

443 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

444 ), 

445 ), 

446 ( 

447 "test_metric_comp.output", 

448 "ingest/run", 

449 "R", 

450 "DummyCamComp", 

451 "d-r", 

452 "423", 

453 testRepo.butler.datastore.root.join( 

454 "ingest/run/test_metric_comp.output/" 

455 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

456 ), 

457 ), 

458 ) 

459 ), 

460 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

461 ), 

462 AstropyTable( 

463 array( 

464 ( 

465 ( 

466 "test_metric_comp.summary", 

467 "foo", 

468 "R", 

469 "DummyCamComp", 

470 "d-r", 

471 "424", 

472 testRepo.butler.datastore.root.join( 

473 "foo/test_metric_comp.summary/" 

474 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

475 ), 

476 ), 

477 ( 

478 "test_metric_comp.summary", 

479 "ingest/run", 

480 "R", 

481 "DummyCamComp", 

482 "d-r", 

483 "423", 

484 testRepo.butler.datastore.root.join( 

485 "ingest/run/test_metric_comp.summary/" 

486 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

487 ), 

488 ), 

489 ) 

490 ), 

491 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

492 ), 

493 ) 

494 

495 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

496 

497 

498if __name__ == "__main__": 498 ↛ 499line 498 didn't jump to line 499, because the condition on line 498 was never true

499 unittest.main()