Coverage for tests/test_cliCmdQueryDatasets.py: 43%
61 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 02:30 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 02:30 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from lsst.resources import ResourcePath
33from numpy import array
35TESTDIR = os.path.abspath(os.path.dirname(__file__))
38def expectedFilesystemDatastoreTables(root: ResourcePath):
39 return (
40 AstropyTable(
41 array(
42 (
43 (
44 "test_metric_comp.data",
45 "ingest/run",
46 "R",
47 "DummyCamComp",
48 "d-r",
49 "423",
50 root.join(
51 "ingest/run/test_metric_comp.data/"
52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
53 ),
54 ),
55 (
56 "test_metric_comp.data",
57 "ingest/run",
58 "R",
59 "DummyCamComp",
60 "d-r",
61 "424",
62 root.join(
63 "ingest/run/test_metric_comp.data/"
64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
65 ),
66 ),
67 )
68 ),
69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
70 ),
71 AstropyTable(
72 array(
73 (
74 (
75 "test_metric_comp.output",
76 "ingest/run",
77 "R",
78 "DummyCamComp",
79 "d-r",
80 "423",
81 root.join(
82 "ingest/run/test_metric_comp.output/"
83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
84 ),
85 ),
86 (
87 "test_metric_comp.output",
88 "ingest/run",
89 "R",
90 "DummyCamComp",
91 "d-r",
92 "424",
93 root.join(
94 "ingest/run/test_metric_comp.output/"
95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
96 ),
97 ),
98 )
99 ),
100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
101 ),
102 AstropyTable(
103 array(
104 (
105 (
106 "test_metric_comp.summary",
107 "ingest/run",
108 "R",
109 "DummyCamComp",
110 "d-r",
111 "423",
112 root.join(
113 "ingest/run/test_metric_comp.summary/"
114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
115 ),
116 ),
117 (
118 "test_metric_comp.summary",
119 "ingest/run",
120 "R",
121 "DummyCamComp",
122 "d-r",
123 "424",
124 root.join(
125 "ingest/run/test_metric_comp.summary/"
126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
127 ),
128 ),
129 )
130 ),
131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
132 ),
133 )
136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
138 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
139 storageClassFactory = StorageClassFactory()
141 @staticmethod
142 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
143 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
145 def setUp(self):
146 self.testdir = makeTestTempDir(TESTDIR)
147 self.repoDir = os.path.join(self.testdir, "repo")
149 def tearDown(self):
150 removeTestTempDir(self.testdir)
152 def testChained(self):
153 root = makeTestTempDir(TESTDIR)
154 testRepo = MetricTestRepo(root, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml"))
156 tables = self._queryDatasets(repo=root, show_uri=True)
158 self.assertAstropyTablesEqual(
159 tables,
160 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root),
161 filterColumns=True,
162 )
164 def testShowURI(self):
165 """Test for expected output with show_uri=True."""
166 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
168 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
170 self.assertAstropyTablesEqual(
171 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True
172 )
174 def testNoShowURI(self):
175 """Test for expected output without show_uri (default is False)."""
176 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
178 tables = self._queryDatasets(repo=self.repoDir)
180 expectedTables = (
181 AstropyTable(
182 array(
183 (
184 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
185 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
186 )
187 ),
188 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
189 ),
190 )
192 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
194 def testWhere(self):
195 """Test using the where clause to reduce the number of rows returned by
196 queryDatasets.
197 """
198 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
200 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
202 expectedTables = (
203 AstropyTable(
204 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")),
205 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
206 ),
207 )
209 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
211 def testGlobDatasetType(self):
212 """Test specifying dataset type."""
213 # Create and register an additional DatasetType
214 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
216 testRepo.butler.registry.insertDimensionData(
217 "visit",
218 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"},
219 )
221 datasetType = addDatasetType(
222 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
223 )
225 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
227 # verify the new dataset type increases the number of tables found:
228 tables = self._queryDatasets(repo=self.repoDir)
230 expectedTables = (
231 AstropyTable(
232 array(
233 (
234 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
235 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
236 )
237 ),
238 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
239 ),
240 AstropyTable(
241 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))),
242 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
243 ),
244 )
246 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
248 def testFindFirstAndCollections(self):
249 """Test the find-first option, and the collections option, since it
250 is required for find-first."""
252 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
254 # Add a new run, and add a dataset to shadow an existing dataset.
255 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
257 # Verify that without find-first, duplicate datasets are returned
258 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
260 expectedTables = (
261 AstropyTable(
262 array(
263 (
264 (
265 "test_metric_comp.data",
266 "foo",
267 "R",
268 "DummyCamComp",
269 "d-r",
270 "424",
271 testRepo.butler.datastore.root.join(
272 "foo/test_metric_comp.data/"
273 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
274 ),
275 ),
276 (
277 "test_metric_comp.data",
278 "ingest/run",
279 "R",
280 "DummyCamComp",
281 "d-r",
282 "423",
283 testRepo.butler.datastore.root.join(
284 "ingest/run/test_metric_comp.data/"
285 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
286 ),
287 ),
288 (
289 "test_metric_comp.data",
290 "ingest/run",
291 "R",
292 "DummyCamComp",
293 "d-r",
294 "424",
295 testRepo.butler.datastore.root.join(
296 "ingest/run/test_metric_comp.data/"
297 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
298 ),
299 ),
300 )
301 ),
302 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
303 ),
304 AstropyTable(
305 array(
306 (
307 (
308 "test_metric_comp.output",
309 "foo",
310 "R",
311 "DummyCamComp",
312 "d-r",
313 "424",
314 testRepo.butler.datastore.root.join(
315 "foo/test_metric_comp.output/"
316 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
317 ),
318 ),
319 (
320 "test_metric_comp.output",
321 "ingest/run",
322 "R",
323 "DummyCamComp",
324 "d-r",
325 "423",
326 testRepo.butler.datastore.root.join(
327 "ingest/run/test_metric_comp.output/"
328 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
329 ),
330 ),
331 (
332 "test_metric_comp.output",
333 "ingest/run",
334 "R",
335 "DummyCamComp",
336 "d-r",
337 "424",
338 testRepo.butler.datastore.root.join(
339 "ingest/run/test_metric_comp.output/"
340 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
341 ),
342 ),
343 )
344 ),
345 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
346 ),
347 AstropyTable(
348 array(
349 (
350 (
351 "test_metric_comp.summary",
352 "foo",
353 "R",
354 "DummyCamComp",
355 "d-r",
356 "424",
357 testRepo.butler.datastore.root.join(
358 "foo/test_metric_comp.summary/"
359 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
360 ),
361 ),
362 (
363 "test_metric_comp.summary",
364 "ingest/run",
365 "R",
366 "DummyCamComp",
367 "d-r",
368 "423",
369 testRepo.butler.datastore.root.join(
370 "ingest/run/test_metric_comp.summary/"
371 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
372 ),
373 ),
374 (
375 "test_metric_comp.summary",
376 "ingest/run",
377 "R",
378 "DummyCamComp",
379 "d-r",
380 "424",
381 testRepo.butler.datastore.root.join(
382 "ingest/run/test_metric_comp.summary/"
383 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
384 ),
385 ),
386 )
387 ),
388 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
389 ),
390 )
392 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
394 # Verify that with find first the duplicate dataset is eliminated and
395 # the more recent dataset is returned.
396 tables = self._queryDatasets(
397 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
398 )
400 expectedTables = (
401 AstropyTable(
402 array(
403 (
404 (
405 "test_metric_comp.data",
406 "foo",
407 "R",
408 "DummyCamComp",
409 "d-r",
410 "424",
411 testRepo.butler.datastore.root.join(
412 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
413 ),
414 ),
415 (
416 "test_metric_comp.data",
417 "ingest/run",
418 "R",
419 "DummyCamComp",
420 "d-r",
421 "423",
422 testRepo.butler.datastore.root.join(
423 "ingest/run/test_metric_comp.data/"
424 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
425 ),
426 ),
427 )
428 ),
429 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
430 ),
431 AstropyTable(
432 array(
433 (
434 (
435 "test_metric_comp.output",
436 "foo",
437 "R",
438 "DummyCamComp",
439 "d-r",
440 "424",
441 testRepo.butler.datastore.root.join(
442 "foo/test_metric_comp.output/"
443 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
444 ),
445 ),
446 (
447 "test_metric_comp.output",
448 "ingest/run",
449 "R",
450 "DummyCamComp",
451 "d-r",
452 "423",
453 testRepo.butler.datastore.root.join(
454 "ingest/run/test_metric_comp.output/"
455 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
456 ),
457 ),
458 )
459 ),
460 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
461 ),
462 AstropyTable(
463 array(
464 (
465 (
466 "test_metric_comp.summary",
467 "foo",
468 "R",
469 "DummyCamComp",
470 "d-r",
471 "424",
472 testRepo.butler.datastore.root.join(
473 "foo/test_metric_comp.summary/"
474 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
475 ),
476 ),
477 (
478 "test_metric_comp.summary",
479 "ingest/run",
480 "R",
481 "DummyCamComp",
482 "d-r",
483 "423",
484 testRepo.butler.datastore.root.join(
485 "ingest/run/test_metric_comp.summary/"
486 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
487 ),
488 ),
489 )
490 ),
491 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
492 ),
493 )
495 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
498if __name__ == "__main__": 498 ↛ 499line 498 didn't jump to line 499, because the condition on line 498 was never true
499 unittest.main()