Coverage for tests/test_cliCmdQueryDatasets.py: 39%
65 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:26 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-05 01:26 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from lsst.resources import ResourcePath
33from numpy import array
35TESTDIR = os.path.abspath(os.path.dirname(__file__))
38def expectedFilesystemDatastoreTables(root: ResourcePath):
39 """Return the expected table contents."""
40 return (
41 AstropyTable(
42 array(
43 (
44 (
45 "test_metric_comp.data",
46 "ingest/run",
47 "R",
48 "DummyCamComp",
49 "d-r",
50 "423",
51 root.join(
52 "ingest/run/test_metric_comp.data/"
53 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
54 ),
55 ),
56 (
57 "test_metric_comp.data",
58 "ingest/run",
59 "R",
60 "DummyCamComp",
61 "d-r",
62 "424",
63 root.join(
64 "ingest/run/test_metric_comp.data/"
65 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
66 ),
67 ),
68 )
69 ),
70 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
71 ),
72 AstropyTable(
73 array(
74 (
75 (
76 "test_metric_comp.output",
77 "ingest/run",
78 "R",
79 "DummyCamComp",
80 "d-r",
81 "423",
82 root.join(
83 "ingest/run/test_metric_comp.output/"
84 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
85 ),
86 ),
87 (
88 "test_metric_comp.output",
89 "ingest/run",
90 "R",
91 "DummyCamComp",
92 "d-r",
93 "424",
94 root.join(
95 "ingest/run/test_metric_comp.output/"
96 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
97 ),
98 ),
99 )
100 ),
101 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
102 ),
103 AstropyTable(
104 array(
105 (
106 (
107 "test_metric_comp.summary",
108 "ingest/run",
109 "R",
110 "DummyCamComp",
111 "d-r",
112 "423",
113 root.join(
114 "ingest/run/test_metric_comp.summary/"
115 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
116 ),
117 ),
118 (
119 "test_metric_comp.summary",
120 "ingest/run",
121 "R",
122 "DummyCamComp",
123 "d-r",
124 "424",
125 root.join(
126 "ingest/run/test_metric_comp.summary/"
127 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
128 ),
129 ),
130 )
131 ),
132 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
133 ),
134 )
137class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
138 """Test the query-datasets command-line."""
140 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
141 storageClassFactory = StorageClassFactory()
143 @staticmethod
144 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
145 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
147 def setUp(self):
148 self.testdir = makeTestTempDir(TESTDIR)
149 self.repoDir = os.path.join(self.testdir, "repo")
151 def tearDown(self):
152 removeTestTempDir(self.testdir)
154 def testChained(self):
155 testRepo = MetricTestRepo(
156 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
157 )
159 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
161 # Want second datastore root.
162 roots = testRepo.butler.get_datastore_roots()
163 datastore_root = roots[testRepo.butler.get_datastore_names()[1]]
165 self.assertAstropyTablesEqual(
166 tables,
167 expectedFilesystemDatastoreTables(datastore_root),
168 filterColumns=True,
169 )
171 def testShowURI(self):
172 """Test for expected output with show_uri=True."""
173 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
175 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
177 roots = testRepo.butler.get_datastore_roots()
178 datastore_root = list(roots.values())[0]
180 self.assertAstropyTablesEqual(
181 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True
182 )
184 def testNoShowURI(self):
185 """Test for expected output without show_uri (default is False)."""
186 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
188 tables = self._queryDatasets(repo=self.repoDir)
190 expectedTables = (
191 AstropyTable(
192 array(
193 (
194 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
195 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
196 )
197 ),
198 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
199 ),
200 )
202 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
204 def testWhere(self):
205 """Test using the where clause to reduce the number of rows returned by
206 queryDatasets.
207 """
208 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
210 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
212 expectedTables = (
213 AstropyTable(
214 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")),
215 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
216 ),
217 )
219 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
221 def testGlobDatasetType(self):
222 """Test specifying dataset type."""
223 # Create and register an additional DatasetType
224 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
226 testRepo.butler.registry.insertDimensionData(
227 "visit",
228 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"},
229 )
231 datasetType = addDatasetType(
232 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
233 )
235 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
237 # verify the new dataset type increases the number of tables found:
238 tables = self._queryDatasets(repo=self.repoDir)
240 expectedTables = (
241 AstropyTable(
242 array(
243 (
244 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
245 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
246 )
247 ),
248 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
249 ),
250 AstropyTable(
251 array(("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425")),
252 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
253 ),
254 )
256 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
258 def testFindFirstAndCollections(self):
259 """Test the find-first option, and the collections option, since it
260 is required for find-first.
261 """
262 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
264 # Add a new run, and add a dataset to shadow an existing dataset.
265 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
267 # Verify that without find-first, duplicate datasets are returned
268 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
270 # The test should be running with a single FileDatastore.
271 roots = testRepo.butler.get_datastore_roots()
272 assert len(roots) == 1
273 datastore_root = list(roots.values())[0]
275 expectedTables = (
276 AstropyTable(
277 array(
278 (
279 (
280 "test_metric_comp.data",
281 "foo",
282 "R",
283 "DummyCamComp",
284 "d-r",
285 "424",
286 datastore_root.join(
287 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
288 ),
289 ),
290 (
291 "test_metric_comp.data",
292 "ingest/run",
293 "R",
294 "DummyCamComp",
295 "d-r",
296 "423",
297 datastore_root.join(
298 "ingest/run/test_metric_comp.data/"
299 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
300 ),
301 ),
302 (
303 "test_metric_comp.data",
304 "ingest/run",
305 "R",
306 "DummyCamComp",
307 "d-r",
308 "424",
309 datastore_root.join(
310 "ingest/run/test_metric_comp.data/"
311 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
312 ),
313 ),
314 )
315 ),
316 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
317 ),
318 AstropyTable(
319 array(
320 (
321 (
322 "test_metric_comp.output",
323 "foo",
324 "R",
325 "DummyCamComp",
326 "d-r",
327 "424",
328 datastore_root.join(
329 "foo/test_metric_comp.output/"
330 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
331 ),
332 ),
333 (
334 "test_metric_comp.output",
335 "ingest/run",
336 "R",
337 "DummyCamComp",
338 "d-r",
339 "423",
340 datastore_root.join(
341 "ingest/run/test_metric_comp.output/"
342 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
343 ),
344 ),
345 (
346 "test_metric_comp.output",
347 "ingest/run",
348 "R",
349 "DummyCamComp",
350 "d-r",
351 "424",
352 datastore_root.join(
353 "ingest/run/test_metric_comp.output/"
354 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
355 ),
356 ),
357 )
358 ),
359 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
360 ),
361 AstropyTable(
362 array(
363 (
364 (
365 "test_metric_comp.summary",
366 "foo",
367 "R",
368 "DummyCamComp",
369 "d-r",
370 "424",
371 datastore_root.join(
372 "foo/test_metric_comp.summary/"
373 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
374 ),
375 ),
376 (
377 "test_metric_comp.summary",
378 "ingest/run",
379 "R",
380 "DummyCamComp",
381 "d-r",
382 "423",
383 datastore_root.join(
384 "ingest/run/test_metric_comp.summary/"
385 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
386 ),
387 ),
388 (
389 "test_metric_comp.summary",
390 "ingest/run",
391 "R",
392 "DummyCamComp",
393 "d-r",
394 "424",
395 datastore_root.join(
396 "ingest/run/test_metric_comp.summary/"
397 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
398 ),
399 ),
400 )
401 ),
402 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
403 ),
404 )
406 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
408 # Verify that with find first the duplicate dataset is eliminated and
409 # the more recent dataset is returned.
410 tables = self._queryDatasets(
411 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
412 )
414 expectedTables = (
415 AstropyTable(
416 array(
417 (
418 (
419 "test_metric_comp.data",
420 "foo",
421 "R",
422 "DummyCamComp",
423 "d-r",
424 "424",
425 datastore_root.join(
426 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
427 ),
428 ),
429 (
430 "test_metric_comp.data",
431 "ingest/run",
432 "R",
433 "DummyCamComp",
434 "d-r",
435 "423",
436 datastore_root.join(
437 "ingest/run/test_metric_comp.data/"
438 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
439 ),
440 ),
441 )
442 ),
443 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
444 ),
445 AstropyTable(
446 array(
447 (
448 (
449 "test_metric_comp.output",
450 "foo",
451 "R",
452 "DummyCamComp",
453 "d-r",
454 "424",
455 datastore_root.join(
456 "foo/test_metric_comp.output/"
457 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
458 ),
459 ),
460 (
461 "test_metric_comp.output",
462 "ingest/run",
463 "R",
464 "DummyCamComp",
465 "d-r",
466 "423",
467 datastore_root.join(
468 "ingest/run/test_metric_comp.output/"
469 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
470 ),
471 ),
472 )
473 ),
474 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
475 ),
476 AstropyTable(
477 array(
478 (
479 (
480 "test_metric_comp.summary",
481 "foo",
482 "R",
483 "DummyCamComp",
484 "d-r",
485 "424",
486 datastore_root.join(
487 "foo/test_metric_comp.summary/"
488 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
489 ),
490 ),
491 (
492 "test_metric_comp.summary",
493 "ingest/run",
494 "R",
495 "DummyCamComp",
496 "d-r",
497 "423",
498 datastore_root.join(
499 "ingest/run/test_metric_comp.summary/"
500 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
501 ),
502 ),
503 )
504 ),
505 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
506 ),
507 )
509 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
512if __name__ == "__main__":
513 unittest.main()