Coverage for tests/test_cliCmdQueryDatasets.py: 24%
103 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-06 01:33 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Unit tests for daf_butler CLI query-datasets command."""
30import os
31import unittest
33from astropy.table import Table as AstropyTable
34from numpy import array
36from lsst.daf.butler import CollectionType, InvalidQueryError, StorageClassFactory, script
37from lsst.daf.butler.tests import addDatasetType
38from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
39from lsst.resources import ResourcePath
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44def expectedFilesystemDatastoreTables(root: ResourcePath):
45 """Return the expected table contents."""
46 return (
47 AstropyTable(
48 array(
49 (
50 (
51 "test_metric_comp.data",
52 "ingest/run",
53 "DummyCamComp",
54 "423",
55 "R",
56 "d-r",
57 root.join(
58 "ingest/run/test_metric_comp.data/"
59 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
60 ),
61 ),
62 (
63 "test_metric_comp.data",
64 "ingest/run",
65 "DummyCamComp",
66 "424",
67 "R",
68 "d-r",
69 root.join(
70 "ingest/run/test_metric_comp.data/"
71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
72 ),
73 ),
74 )
75 ),
76 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
77 ),
78 AstropyTable(
79 array(
80 (
81 (
82 "test_metric_comp.output",
83 "ingest/run",
84 "DummyCamComp",
85 "423",
86 "R",
87 "d-r",
88 root.join(
89 "ingest/run/test_metric_comp.output/"
90 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
91 ),
92 ),
93 (
94 "test_metric_comp.output",
95 "ingest/run",
96 "DummyCamComp",
97 "424",
98 "R",
99 "d-r",
100 root.join(
101 "ingest/run/test_metric_comp.output/"
102 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
103 ),
104 ),
105 )
106 ),
107 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
108 ),
109 AstropyTable(
110 array(
111 (
112 (
113 "test_metric_comp.summary",
114 "ingest/run",
115 "DummyCamComp",
116 "423",
117 "R",
118 "d-r",
119 root.join(
120 "ingest/run/test_metric_comp.summary/"
121 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
122 ),
123 ),
124 (
125 "test_metric_comp.summary",
126 "ingest/run",
127 "DummyCamComp",
128 "424",
129 "R",
130 "d-r",
131 root.join(
132 "ingest/run/test_metric_comp.summary/"
133 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
134 ),
135 ),
136 )
137 ),
138 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
139 ),
140 )
143class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
144 """Test the query-datasets command-line."""
146 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
147 storageClassFactory = StorageClassFactory()
149 @staticmethod
150 def _queryDatasets(
151 repo, glob=(), collections=(), where="", find_first=False, show_uri=False, limit=0, order_by=()
152 ):
153 query = script.QueryDatasets(
154 glob,
155 collections,
156 where=where,
157 find_first=find_first,
158 show_uri=show_uri,
159 limit=limit,
160 order_by=order_by,
161 butler=repo,
162 )
163 return list(query.getTables())
165 def setUp(self):
166 self.testdir = makeTestTempDir(TESTDIR)
167 self.repoDir = os.path.join(self.testdir, "repo")
169 def tearDown(self):
170 removeTestTempDir(self.testdir)
172 def testChained(self):
173 testRepo = MetricTestRepo(
174 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
175 )
177 tables = self._queryDatasets(repo=testRepo.butler, show_uri=True, collections="*", glob="*")
179 # Want second datastore root since in-memory is ephemeral and
180 # all the relevant datasets are stored in the second as well as third
181 # datastore.
182 roots = testRepo.butler.get_datastore_roots()
183 datastore_root = roots[testRepo.butler.get_datastore_names()[1]]
185 self.assertAstropyTablesEqual(
186 tables,
187 expectedFilesystemDatastoreTables(datastore_root),
188 filterColumns=True,
189 )
191 def testShowURI(self):
192 """Test for expected output with show_uri=True."""
193 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
195 tables = self._queryDatasets(repo=testRepo.butler, show_uri=True, collections="*", glob="*")
197 roots = testRepo.butler.get_datastore_roots()
198 datastore_root = list(roots.values())[0]
200 self.assertAstropyTablesEqual(
201 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True
202 )
204 def testShowUriNoDisassembly(self):
205 """Test for expected output with show_uri=True and no disassembly."""
206 testRepo = MetricTestRepo(
207 self.repoDir,
208 configFile=self.configFile,
209 storageClassName="StructuredCompositeReadCompNoDisassembly",
210 )
212 tables = self._queryDatasets(repo=testRepo.butler, show_uri=True, collections="*", glob="*")
214 roots = testRepo.butler.get_datastore_roots()
215 datastore_root = list(roots.values())[0]
217 expected = [
218 AstropyTable(
219 array(
220 (
221 (
222 "test_metric_comp",
223 "ingest/run",
224 "DummyCamComp",
225 "423",
226 "R",
227 "d-r",
228 datastore_root.join(
229 "ingest/run/test_metric_comp/test_metric_comp_v00000423_fDummyCamComp.yaml"
230 ),
231 ),
232 (
233 "test_metric_comp",
234 "ingest/run",
235 "DummyCamComp",
236 "424",
237 "R",
238 "d-r",
239 datastore_root.join(
240 "ingest/run/test_metric_comp/test_metric_comp_v00000424_fDummyCamComp.yaml"
241 ),
242 ),
243 )
244 ),
245 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
246 ),
247 ]
249 self.assertAstropyTablesEqual(tables, expected, filterColumns=True)
251 def testNoShowURI(self):
252 """Test for expected output without show_uri (default is False)."""
253 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
255 tables = self._queryDatasets(repo=testRepo.butler, collections="*", glob="*")
257 expectedTables = (
258 AstropyTable(
259 array(
260 (
261 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
262 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
263 )
264 ),
265 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
266 ),
267 )
269 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
271 def testWhere(self):
272 """Test using the where clause to reduce the number of rows returned by
273 queryDatasets.
274 """
275 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
277 for glob in (("*",), ("test_metric_comp",)):
278 with self.subTest(glob=glob):
279 tables = self._queryDatasets(
280 repo=testRepo.butler,
281 where="instrument='DummyCamComp' AND visit=423",
282 collections="*",
283 glob=glob,
284 )
286 expectedTables = (
287 AstropyTable(
288 array(("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r")),
289 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
290 ),
291 )
293 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
295 with self.assertRaises(InvalidQueryError):
296 self._queryDatasets(repo=testRepo.butler, collections="*", find_first=True, glob=glob)
298 def testGlobDatasetType(self):
299 """Test specifying dataset type."""
300 # Create and register an additional DatasetType
301 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
303 testRepo.butler.registry.insertDimensionData(
304 "visit",
305 {
306 "instrument": "DummyCamComp",
307 "id": 425,
308 "name": "fourtwentyfive",
309 "physical_filter": "d-r",
310 "day_obs": 20200101,
311 },
312 )
314 datasetType = addDatasetType(
315 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
316 )
318 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
320 # verify the new dataset type increases the number of tables found:
321 tables = self._queryDatasets(repo=testRepo.butler, collections="*", glob="*")
323 expectedTables = (
324 AstropyTable(
325 array(("alt_test_metric_comp", "ingest/run", "DummyCamComp", "425", "R", "d-r")),
326 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
327 ),
328 AstropyTable(
329 array(
330 (
331 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
332 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
333 )
334 ),
335 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
336 ),
337 )
339 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
341 # Dataset type (glob) argument will become mandatory soon
342 with self.assertWarns(FutureWarning):
343 self._queryDatasets(repo=testRepo.butler, collections="*")
345 def test_limit_order(self):
346 """Test limit and ordering."""
347 # Create and register an additional DatasetType
348 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
350 with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
351 tables = self._queryDatasets(
352 repo=testRepo.butler,
353 limit=-1,
354 order_by=("visit",),
355 collections="*",
356 glob=("test_metric_comp",),
357 )
359 self.assertIn("increase this limit", cm.output[0])
361 expectedTables = [
362 AstropyTable(
363 array((("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),)),
364 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
365 ),
366 ]
367 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
369 # Same as previous test, but with positive limit so no warning is
370 # issued.
371 with self.assertNoLogs("lsst.daf.butler.script.queryDatasets", level="WARNING"):
372 tables = self._queryDatasets(
373 repo=testRepo.butler,
374 limit=1,
375 order_by=("visit",),
376 collections="*",
377 glob=("test_metric_comp",),
378 )
379 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
381 with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
382 tables = self._queryDatasets(
383 repo=testRepo.butler,
384 limit=-1,
385 order_by=("-visit",),
386 collections="*",
387 glob=("test_metric_comp",),
388 )
389 self.assertIn("increase this limit", cm.output[0])
391 expectedTables = [
392 AstropyTable(
393 array((("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),)),
394 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
395 ),
396 ]
397 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
399 # --order-by is not supported by the query backend for multiple dataset
400 # types, so we can only provide it for queries with a single dataset
401 # type.
402 with self.assertRaisesRegex(NotImplementedError, "--order-by"):
403 self._queryDatasets(
404 repo=testRepo.butler, limit=1, order_by=("visit",), collections="*", glob=["*"]
405 )
406 with self.assertRaisesRegex(NotImplementedError, "--order-by"):
407 self._queryDatasets(
408 repo=testRepo.butler,
409 limit=1,
410 order_by=("visit",),
411 collections="*",
412 glob=["test_metric_comp", "raw"],
413 )
415 def testFindFirstAndCollections(self):
416 """Test the find-first option, and the collections option, since it
417 is required for find-first.
418 """
419 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
421 # Add a new run, and add a dataset to shadow an existing dataset.
422 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
424 # Add a CHAINED collection to include the two runs, to check that
425 # flattening works as well.
426 testRepo.butler.collections.register("chain", CollectionType.CHAINED)
427 testRepo.butler.collections.redefine_chain("chain", ["foo", "ingest/run"])
429 # Verify that without find-first, duplicate datasets are returned
430 tables = self._queryDatasets(repo=testRepo.butler, collections=["chain"], show_uri=True, glob="*")
432 # The test should be running with a single FileDatastore.
433 roots = testRepo.butler.get_datastore_roots()
434 assert len(roots) == 1
435 datastore_root = list(roots.values())[0]
437 expectedTables = (
438 AstropyTable(
439 array(
440 (
441 (
442 "test_metric_comp.data",
443 "foo",
444 "DummyCamComp",
445 "424",
446 "R",
447 "d-r",
448 datastore_root.join(
449 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
450 ),
451 ),
452 (
453 "test_metric_comp.data",
454 "ingest/run",
455 "DummyCamComp",
456 "423",
457 "R",
458 "d-r",
459 datastore_root.join(
460 "ingest/run/test_metric_comp.data/"
461 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
462 ),
463 ),
464 (
465 "test_metric_comp.data",
466 "ingest/run",
467 "DummyCamComp",
468 "424",
469 "R",
470 "d-r",
471 datastore_root.join(
472 "ingest/run/test_metric_comp.data/"
473 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
474 ),
475 ),
476 )
477 ),
478 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
479 ),
480 AstropyTable(
481 array(
482 (
483 (
484 "test_metric_comp.output",
485 "foo",
486 "DummyCamComp",
487 "424",
488 "R",
489 "d-r",
490 datastore_root.join(
491 "foo/test_metric_comp.output/"
492 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
493 ),
494 ),
495 (
496 "test_metric_comp.output",
497 "ingest/run",
498 "DummyCamComp",
499 "423",
500 "R",
501 "d-r",
502 datastore_root.join(
503 "ingest/run/test_metric_comp.output/"
504 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
505 ),
506 ),
507 (
508 "test_metric_comp.output",
509 "ingest/run",
510 "DummyCamComp",
511 "424",
512 "R",
513 "d-r",
514 datastore_root.join(
515 "ingest/run/test_metric_comp.output/"
516 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
517 ),
518 ),
519 )
520 ),
521 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
522 ),
523 AstropyTable(
524 array(
525 (
526 (
527 "test_metric_comp.summary",
528 "foo",
529 "DummyCamComp",
530 "424",
531 "R",
532 "d-r",
533 datastore_root.join(
534 "foo/test_metric_comp.summary/"
535 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
536 ),
537 ),
538 (
539 "test_metric_comp.summary",
540 "ingest/run",
541 "DummyCamComp",
542 "423",
543 "R",
544 "d-r",
545 datastore_root.join(
546 "ingest/run/test_metric_comp.summary/"
547 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
548 ),
549 ),
550 (
551 "test_metric_comp.summary",
552 "ingest/run",
553 "DummyCamComp",
554 "424",
555 "R",
556 "d-r",
557 datastore_root.join(
558 "ingest/run/test_metric_comp.summary/"
559 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
560 ),
561 ),
562 )
563 ),
564 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
565 ),
566 )
568 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
570 # Verify that with find first the duplicate dataset is eliminated and
571 # the more recent dataset is returned.
572 tables = self._queryDatasets(
573 repo=testRepo.butler, collections=["chain"], show_uri=True, find_first=True, glob="*"
574 )
576 expectedTables = (
577 AstropyTable(
578 array(
579 (
580 (
581 "test_metric_comp.data",
582 "foo",
583 "DummyCamComp",
584 "424",
585 "R",
586 "d-r",
587 datastore_root.join(
588 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
589 ),
590 ),
591 (
592 "test_metric_comp.data",
593 "ingest/run",
594 "DummyCamComp",
595 "423",
596 "R",
597 "d-r",
598 datastore_root.join(
599 "ingest/run/test_metric_comp.data/"
600 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
601 ),
602 ),
603 )
604 ),
605 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
606 ),
607 AstropyTable(
608 array(
609 (
610 (
611 "test_metric_comp.output",
612 "foo",
613 "DummyCamComp",
614 "424",
615 "R",
616 "d-r",
617 datastore_root.join(
618 "foo/test_metric_comp.output/"
619 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
620 ),
621 ),
622 (
623 "test_metric_comp.output",
624 "ingest/run",
625 "DummyCamComp",
626 "423",
627 "R",
628 "d-r",
629 datastore_root.join(
630 "ingest/run/test_metric_comp.output/"
631 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
632 ),
633 ),
634 )
635 ),
636 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
637 ),
638 AstropyTable(
639 array(
640 (
641 (
642 "test_metric_comp.summary",
643 "foo",
644 "DummyCamComp",
645 "424",
646 "R",
647 "d-r",
648 datastore_root.join(
649 "foo/test_metric_comp.summary/"
650 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
651 ),
652 ),
653 (
654 "test_metric_comp.summary",
655 "ingest/run",
656 "DummyCamComp",
657 "423",
658 "R",
659 "d-r",
660 datastore_root.join(
661 "ingest/run/test_metric_comp.summary/"
662 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
663 ),
664 ),
665 )
666 ),
667 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
668 ),
669 )
671 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
673 # Verify that globs are not supported with find_first=True.
674 with self.assertRaises(InvalidQueryError):
675 self._queryDatasets(
676 repo=testRepo.butler, collections=["*"], show_uri=True, find_first=True, glob="*"
677 )
679 # Collections argument will become mandatory soon
680 with self.assertWarns(FutureWarning):
681 self._queryDatasets(repo=testRepo.butler, glob="*")
684if __name__ == "__main__":
685 unittest.main()