Coverage for tests/test_cliCmdQueryDatasets.py: 41%

61 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-02 18:18 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-datasets command. 

23""" 

24 

25import os 

26import unittest 

27 

28from astropy.table import Table as AstropyTable 

29from lsst.daf.butler import StorageClassFactory, script 

30from lsst.daf.butler.tests import addDatasetType 

31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir 

32from lsst.resources import ResourcePath 

33from numpy import array 

34 

35TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def expectedFilesystemDatastoreTables(root: ResourcePath): 

39 return ( 

40 AstropyTable( 

41 array( 

42 ( 

43 ( 

44 "test_metric_comp.data", 

45 "ingest/run", 

46 "R", 

47 "DummyCamComp", 

48 "d-r", 

49 "423", 

50 root.join( 

51 "ingest/run/test_metric_comp.data/" 

52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

53 ), 

54 ), 

55 ( 

56 "test_metric_comp.data", 

57 "ingest/run", 

58 "R", 

59 "DummyCamComp", 

60 "d-r", 

61 "424", 

62 root.join( 

63 "ingest/run/test_metric_comp.data/" 

64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

65 ), 

66 ), 

67 ) 

68 ), 

69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

70 ), 

71 AstropyTable( 

72 array( 

73 ( 

74 ( 

75 "test_metric_comp.output", 

76 "ingest/run", 

77 "R", 

78 "DummyCamComp", 

79 "d-r", 

80 "423", 

81 root.join( 

82 "ingest/run/test_metric_comp.output/" 

83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

84 ), 

85 ), 

86 ( 

87 "test_metric_comp.output", 

88 "ingest/run", 

89 "R", 

90 "DummyCamComp", 

91 "d-r", 

92 "424", 

93 root.join( 

94 "ingest/run/test_metric_comp.output/" 

95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

96 ), 

97 ), 

98 ) 

99 ), 

100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

101 ), 

102 AstropyTable( 

103 array( 

104 ( 

105 ( 

106 "test_metric_comp.summary", 

107 "ingest/run", 

108 "R", 

109 "DummyCamComp", 

110 "d-r", 

111 "423", 

112 root.join( 

113 "ingest/run/test_metric_comp.summary/" 

114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

115 ), 

116 ), 

117 ( 

118 "test_metric_comp.summary", 

119 "ingest/run", 

120 "R", 

121 "DummyCamComp", 

122 "d-r", 

123 "424", 

124 root.join( 

125 "ingest/run/test_metric_comp.summary/" 

126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

127 ), 

128 ), 

129 ) 

130 ), 

131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

132 ), 

133 ) 

134 

135 

136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

137 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

138 storageClassFactory = StorageClassFactory() 

139 

140 @staticmethod 

141 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

142 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables() 

143 

144 def setUp(self): 

145 self.testdir = makeTestTempDir(TESTDIR) 

146 self.repoDir = os.path.join(self.testdir, "repo") 

147 

148 def tearDown(self): 

149 removeTestTempDir(self.testdir) 

150 

151 def testChained(self): 

152 root = makeTestTempDir(TESTDIR) 

153 testRepo = MetricTestRepo(root, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")) 

154 

155 tables = self._queryDatasets(repo=root, show_uri=True) 

156 

157 self.assertAstropyTablesEqual( 

158 tables, 

159 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root), 

160 filterColumns=True, 

161 ) 

162 

163 def testShowURI(self): 

164 """Test for expected output with show_uri=True.""" 

165 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

166 

167 tables = self._queryDatasets(repo=self.repoDir, show_uri=True) 

168 

169 self.assertAstropyTablesEqual( 

170 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True 

171 ) 

172 

173 def testNoShowURI(self): 

174 """Test for expected output without show_uri (default is False).""" 

175 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

176 

177 tables = self._queryDatasets(repo=self.repoDir) 

178 

179 expectedTables = ( 

180 AstropyTable( 

181 array( 

182 ( 

183 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

184 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

185 ) 

186 ), 

187 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

188 ), 

189 ) 

190 

191 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

192 

193 def testWhere(self): 

194 """Test using the where clause to reduce the number of rows returned by 

195 queryDatasets. 

196 """ 

197 _ = MetricTestRepo(self.repoDir, configFile=self.configFile) 

198 

199 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423") 

200 

201 expectedTables = ( 

202 AstropyTable( 

203 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")), 

204 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

205 ), 

206 ) 

207 

208 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

209 

210 def testGlobDatasetType(self): 

211 """Test specifying dataset type.""" 

212 # Create and register an additional DatasetType 

213 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

214 

215 testRepo.butler.registry.insertDimensionData( 

216 "visit", 

217 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"}, 

218 ) 

219 

220 datasetType = addDatasetType( 

221 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp" 

222 ) 

223 

224 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

225 

226 # verify the new dataset type increases the number of tables found: 

227 tables = self._queryDatasets(repo=self.repoDir) 

228 

229 expectedTables = ( 

230 AstropyTable( 

231 array( 

232 ( 

233 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"), 

234 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"), 

235 ) 

236 ), 

237 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

238 ), 

239 AstropyTable( 

240 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))), 

241 names=("type", "run", "band", "instrument", "physical_filter", "visit"), 

242 ), 

243 ) 

244 

245 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

246 

247 def testFindFirstAndCollections(self): 

248 """Test the find-first option, and the collections option, since it 

249 is required for find-first.""" 

250 

251 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile) 

252 

253 # Add a new run, and add a dataset to shadow an existing dataset. 

254 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) 

255 

256 # Verify that without find-first, duplicate datasets are returned 

257 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True) 

258 

259 expectedTables = ( 

260 AstropyTable( 

261 array( 

262 ( 

263 ( 

264 "test_metric_comp.data", 

265 "foo", 

266 "R", 

267 "DummyCamComp", 

268 "d-r", 

269 "424", 

270 testRepo.butler.datastore.root.join( 

271 "foo/test_metric_comp.data/" 

272 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

273 ), 

274 ), 

275 ( 

276 "test_metric_comp.data", 

277 "ingest/run", 

278 "R", 

279 "DummyCamComp", 

280 "d-r", 

281 "423", 

282 testRepo.butler.datastore.root.join( 

283 "ingest/run/test_metric_comp.data/" 

284 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

285 ), 

286 ), 

287 ( 

288 "test_metric_comp.data", 

289 "ingest/run", 

290 "R", 

291 "DummyCamComp", 

292 "d-r", 

293 "424", 

294 testRepo.butler.datastore.root.join( 

295 "ingest/run/test_metric_comp.data/" 

296 "test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

297 ), 

298 ), 

299 ) 

300 ), 

301 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

302 ), 

303 AstropyTable( 

304 array( 

305 ( 

306 ( 

307 "test_metric_comp.output", 

308 "foo", 

309 "R", 

310 "DummyCamComp", 

311 "d-r", 

312 "424", 

313 testRepo.butler.datastore.root.join( 

314 "foo/test_metric_comp.output/" 

315 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

316 ), 

317 ), 

318 ( 

319 "test_metric_comp.output", 

320 "ingest/run", 

321 "R", 

322 "DummyCamComp", 

323 "d-r", 

324 "423", 

325 testRepo.butler.datastore.root.join( 

326 "ingest/run/test_metric_comp.output/" 

327 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

328 ), 

329 ), 

330 ( 

331 "test_metric_comp.output", 

332 "ingest/run", 

333 "R", 

334 "DummyCamComp", 

335 "d-r", 

336 "424", 

337 testRepo.butler.datastore.root.join( 

338 "ingest/run/test_metric_comp.output/" 

339 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

340 ), 

341 ), 

342 ) 

343 ), 

344 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

345 ), 

346 AstropyTable( 

347 array( 

348 ( 

349 ( 

350 "test_metric_comp.summary", 

351 "foo", 

352 "R", 

353 "DummyCamComp", 

354 "d-r", 

355 "424", 

356 testRepo.butler.datastore.root.join( 

357 "foo/test_metric_comp.summary/" 

358 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

359 ), 

360 ), 

361 ( 

362 "test_metric_comp.summary", 

363 "ingest/run", 

364 "R", 

365 "DummyCamComp", 

366 "d-r", 

367 "423", 

368 testRepo.butler.datastore.root.join( 

369 "ingest/run/test_metric_comp.summary/" 

370 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

371 ), 

372 ), 

373 ( 

374 "test_metric_comp.summary", 

375 "ingest/run", 

376 "R", 

377 "DummyCamComp", 

378 "d-r", 

379 "424", 

380 testRepo.butler.datastore.root.join( 

381 "ingest/run/test_metric_comp.summary/" 

382 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

383 ), 

384 ), 

385 ) 

386 ), 

387 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

388 ), 

389 ) 

390 

391 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

392 

393 # Verify that with find first the duplicate dataset is eliminated and 

394 # the more recent dataset is returned. 

395 tables = self._queryDatasets( 

396 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True 

397 ) 

398 

399 expectedTables = ( 

400 AstropyTable( 

401 array( 

402 ( 

403 ( 

404 "test_metric_comp.data", 

405 "foo", 

406 "R", 

407 "DummyCamComp", 

408 "d-r", 

409 "424", 

410 testRepo.butler.datastore.root.join( 

411 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml" 

412 ), 

413 ), 

414 ( 

415 "test_metric_comp.data", 

416 "ingest/run", 

417 "R", 

418 "DummyCamComp", 

419 "d-r", 

420 "423", 

421 testRepo.butler.datastore.root.join( 

422 "ingest/run/test_metric_comp.data/" 

423 "test_metric_comp_v00000423_fDummyCamComp_data.yaml" 

424 ), 

425 ), 

426 ) 

427 ), 

428 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

429 ), 

430 AstropyTable( 

431 array( 

432 ( 

433 ( 

434 "test_metric_comp.output", 

435 "foo", 

436 "R", 

437 "DummyCamComp", 

438 "d-r", 

439 "424", 

440 testRepo.butler.datastore.root.join( 

441 "foo/test_metric_comp.output/" 

442 "test_metric_comp_v00000424_fDummyCamComp_output.yaml" 

443 ), 

444 ), 

445 ( 

446 "test_metric_comp.output", 

447 "ingest/run", 

448 "R", 

449 "DummyCamComp", 

450 "d-r", 

451 "423", 

452 testRepo.butler.datastore.root.join( 

453 "ingest/run/test_metric_comp.output/" 

454 "test_metric_comp_v00000423_fDummyCamComp_output.yaml" 

455 ), 

456 ), 

457 ) 

458 ), 

459 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

460 ), 

461 AstropyTable( 

462 array( 

463 ( 

464 ( 

465 "test_metric_comp.summary", 

466 "foo", 

467 "R", 

468 "DummyCamComp", 

469 "d-r", 

470 "424", 

471 testRepo.butler.datastore.root.join( 

472 "foo/test_metric_comp.summary/" 

473 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml" 

474 ), 

475 ), 

476 ( 

477 "test_metric_comp.summary", 

478 "ingest/run", 

479 "R", 

480 "DummyCamComp", 

481 "d-r", 

482 "423", 

483 testRepo.butler.datastore.root.join( 

484 "ingest/run/test_metric_comp.summary/" 

485 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml" 

486 ), 

487 ), 

488 ) 

489 ), 

490 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"), 

491 ), 

492 ) 

493 

494 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True) 

495 

496 

497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true

498 unittest.main()