Coverage for tests/test_cliCmdQueryDatasets.py: 22%
111 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-21 01:20 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Unit tests for daf_butler CLI query-datasets command."""
30import os
31import unittest
33from astropy.table import Table as AstropyTable
34from numpy import array
36from lsst.daf.butler import CollectionType, InvalidQueryError, StorageClassFactory, script
37from lsst.daf.butler.tests import addDatasetType
38from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
39from lsst.resources import ResourcePath
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44def expectedFilesystemDatastoreTables(root: ResourcePath):
45 """Return the expected table contents."""
46 return (
47 AstropyTable(
48 array(
49 (
50 (
51 "test_metric_comp.data",
52 "ingest/run",
53 "DummyCamComp",
54 "423",
55 "R",
56 "d-r",
57 root.join(
58 "ingest/run/test_metric_comp.data/"
59 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
60 ),
61 ),
62 (
63 "test_metric_comp.data",
64 "ingest/run",
65 "DummyCamComp",
66 "424",
67 "R",
68 "d-r",
69 root.join(
70 "ingest/run/test_metric_comp.data/"
71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
72 ),
73 ),
74 )
75 ),
76 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
77 ),
78 AstropyTable(
79 array(
80 (
81 (
82 "test_metric_comp.output",
83 "ingest/run",
84 "DummyCamComp",
85 "423",
86 "R",
87 "d-r",
88 root.join(
89 "ingest/run/test_metric_comp.output/"
90 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
91 ),
92 ),
93 (
94 "test_metric_comp.output",
95 "ingest/run",
96 "DummyCamComp",
97 "424",
98 "R",
99 "d-r",
100 root.join(
101 "ingest/run/test_metric_comp.output/"
102 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
103 ),
104 ),
105 )
106 ),
107 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
108 ),
109 AstropyTable(
110 array(
111 (
112 (
113 "test_metric_comp.summary",
114 "ingest/run",
115 "DummyCamComp",
116 "423",
117 "R",
118 "d-r",
119 root.join(
120 "ingest/run/test_metric_comp.summary/"
121 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
122 ),
123 ),
124 (
125 "test_metric_comp.summary",
126 "ingest/run",
127 "DummyCamComp",
128 "424",
129 "R",
130 "d-r",
131 root.join(
132 "ingest/run/test_metric_comp.summary/"
133 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
134 ),
135 ),
136 )
137 ),
138 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
139 ),
140 )
143class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
144 """Test the query-datasets command-line."""
146 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
147 storageClassFactory = StorageClassFactory()
149 @staticmethod
150 def _queryDatasets(
151 repo, glob=(), collections=(), where="", find_first=False, show_uri=False, limit=0, order_by=()
152 ):
153 query = script.QueryDatasets(
154 glob,
155 collections,
156 where=where,
157 find_first=find_first,
158 show_uri=show_uri,
159 limit=limit,
160 order_by=order_by,
161 butler=repo,
162 )
163 return list(query.getTables())
165 def setUp(self):
166 self.testdir = makeTestTempDir(TESTDIR)
167 self.repoDir = os.path.join(self.testdir, "repo")
169 def tearDown(self):
170 removeTestTempDir(self.testdir)
172 def testChained(self):
173 testRepo = MetricTestRepo(
174 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
175 )
176 self.enterContext(testRepo.butler)
178 tables = self._queryDatasets(repo=testRepo.butler, show_uri=True, collections="*", glob="*")
180 # Want second datastore root since in-memory is ephemeral and
181 # all the relevant datasets are stored in the second as well as third
182 # datastore.
183 roots = testRepo.butler.get_datastore_roots()
184 datastore_root = roots[testRepo.butler.get_datastore_names()[1]]
186 self.assertAstropyTablesEqual(
187 tables,
188 expectedFilesystemDatastoreTables(datastore_root),
189 filterColumns=True,
190 )
192 def testShowURI(self):
193 """Test for expected output with show_uri=True."""
194 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
195 self.enterContext(testRepo.butler)
197 tables = self._queryDatasets(repo=testRepo.butler, show_uri=True, collections="*", glob="*")
199 roots = testRepo.butler.get_datastore_roots()
200 datastore_root = list(roots.values())[0]
202 self.assertAstropyTablesEqual(
203 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True
204 )
206 def testShowUriNoDisassembly(self):
207 """Test for expected output with show_uri=True and no disassembly."""
208 testRepo = MetricTestRepo(
209 self.repoDir,
210 configFile=self.configFile,
211 storageClassName="StructuredCompositeReadCompNoDisassembly",
212 )
213 self.enterContext(testRepo.butler)
215 tables = self._queryDatasets(repo=testRepo.butler, show_uri=True, collections="*", glob="*")
217 roots = testRepo.butler.get_datastore_roots()
218 datastore_root = list(roots.values())[0]
220 expected = [
221 AstropyTable(
222 array(
223 (
224 (
225 "test_metric_comp",
226 "ingest/run",
227 "DummyCamComp",
228 "423",
229 "R",
230 "d-r",
231 datastore_root.join(
232 "ingest/run/test_metric_comp/test_metric_comp_v00000423_fDummyCamComp.yaml"
233 ),
234 ),
235 (
236 "test_metric_comp",
237 "ingest/run",
238 "DummyCamComp",
239 "424",
240 "R",
241 "d-r",
242 datastore_root.join(
243 "ingest/run/test_metric_comp/test_metric_comp_v00000424_fDummyCamComp.yaml"
244 ),
245 ),
246 )
247 ),
248 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
249 ),
250 ]
252 self.assertAstropyTablesEqual(tables, expected, filterColumns=True)
254 def testNoShowURI(self):
255 """Test for expected output without show_uri (default is False)."""
256 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
257 self.enterContext(testRepo.butler)
259 tables = self._queryDatasets(repo=testRepo.butler, collections="*", glob="*")
261 expectedTables = (
262 AstropyTable(
263 array(
264 (
265 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
266 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
267 )
268 ),
269 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
270 ),
271 )
273 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
275 def testWhere(self):
276 """Test using the where clause to reduce the number of rows returned by
277 queryDatasets.
278 """
279 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
280 self.enterContext(testRepo.butler)
282 for glob in (("*",), ("test_metric_comp",)):
283 with self.subTest(glob=glob):
284 tables = self._queryDatasets(
285 repo=testRepo.butler,
286 where="instrument='DummyCamComp' AND visit=423",
287 collections="*",
288 glob=glob,
289 )
291 expectedTables = (
292 AstropyTable(
293 array(("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r")),
294 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
295 ),
296 )
298 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
300 with self.assertRaises(InvalidQueryError):
301 self._queryDatasets(repo=testRepo.butler, collections="*", find_first=True, glob=glob)
303 def testGlobDatasetType(self):
304 """Test specifying dataset type."""
305 # Create and register an additional DatasetType
306 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
307 self.enterContext(testRepo.butler)
309 testRepo.butler.registry.insertDimensionData(
310 "visit",
311 {
312 "instrument": "DummyCamComp",
313 "id": 425,
314 "name": "fourtwentyfive",
315 "physical_filter": "d-r",
316 "day_obs": 20200101,
317 },
318 )
320 datasetType = addDatasetType(
321 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
322 )
324 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
326 # verify the new dataset type increases the number of tables found:
327 tables = self._queryDatasets(repo=testRepo.butler, collections="*", glob="*")
329 expectedTables = (
330 AstropyTable(
331 array(("alt_test_metric_comp", "ingest/run", "DummyCamComp", "425", "R", "d-r")),
332 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
333 ),
334 AstropyTable(
335 array(
336 (
337 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
338 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
339 )
340 ),
341 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
342 ),
343 )
345 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
347 # Dataset type (glob) argument will become mandatory soon
348 with self.assertWarns(FutureWarning):
349 self._queryDatasets(repo=testRepo.butler, collections="*")
351 def test_limit_order(self):
352 """Test limit and ordering."""
353 # Create and register an additional DatasetType
354 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
355 self.enterContext(testRepo.butler)
357 with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
358 tables = self._queryDatasets(
359 repo=testRepo.butler,
360 limit=-1,
361 order_by=("visit",),
362 collections="*",
363 glob=("test_metric_comp",),
364 )
366 self.assertIn("increase this limit", cm.output[0])
368 expectedTables = [
369 AstropyTable(
370 array((("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),)),
371 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
372 ),
373 ]
374 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
376 # Same as previous test, but with positive limit so no warning is
377 # issued.
378 with self.assertNoLogs("lsst.daf.butler.script.queryDatasets", level="WARNING"):
379 tables = self._queryDatasets(
380 repo=testRepo.butler,
381 limit=1,
382 order_by=("visit",),
383 collections="*",
384 glob=("test_metric_comp",),
385 )
386 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
388 with self.assertLogs("lsst.daf.butler.script.queryDatasets", level="WARNING") as cm:
389 tables = self._queryDatasets(
390 repo=testRepo.butler,
391 limit=-1,
392 order_by=("-visit",),
393 collections="*",
394 glob=("test_metric_comp",),
395 )
396 self.assertIn("increase this limit", cm.output[0])
398 expectedTables = [
399 AstropyTable(
400 array((("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),)),
401 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
402 ),
403 ]
404 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
406 # --order-by is not supported by the query backend for multiple dataset
407 # types, so we can only provide it for queries with a single dataset
408 # type.
409 with self.assertRaisesRegex(NotImplementedError, "--order-by"):
410 self._queryDatasets(
411 repo=testRepo.butler, limit=1, order_by=("visit",), collections="*", glob=["*"]
412 )
413 with self.assertRaisesRegex(NotImplementedError, "--order-by"):
414 self._queryDatasets(
415 repo=testRepo.butler,
416 limit=1,
417 order_by=("visit",),
418 collections="*",
419 glob=["test_metric_comp", "raw"],
420 )
422 def testFindFirstAndCollections(self):
423 """Test the find-first option, and the collections option, since it
424 is required for find-first.
425 """
426 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
427 self.enterContext(testRepo.butler)
429 # Add a new run, and add a dataset to shadow an existing dataset.
430 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
432 # Add a CHAINED collection to include the two runs, to check that
433 # flattening works as well.
434 testRepo.butler.collections.register("chain", CollectionType.CHAINED)
435 testRepo.butler.collections.redefine_chain("chain", ["foo", "ingest/run"])
437 # Verify that without find-first, duplicate datasets are returned
438 tables = self._queryDatasets(repo=testRepo.butler, collections=["chain"], show_uri=True, glob="*")
440 # The test should be running with a single FileDatastore.
441 roots = testRepo.butler.get_datastore_roots()
442 assert len(roots) == 1
443 datastore_root = list(roots.values())[0]
445 expectedTables = (
446 AstropyTable(
447 array(
448 (
449 (
450 "test_metric_comp.data",
451 "foo",
452 "DummyCamComp",
453 "424",
454 "R",
455 "d-r",
456 datastore_root.join(
457 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
458 ),
459 ),
460 (
461 "test_metric_comp.data",
462 "ingest/run",
463 "DummyCamComp",
464 "423",
465 "R",
466 "d-r",
467 datastore_root.join(
468 "ingest/run/test_metric_comp.data/"
469 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
470 ),
471 ),
472 (
473 "test_metric_comp.data",
474 "ingest/run",
475 "DummyCamComp",
476 "424",
477 "R",
478 "d-r",
479 datastore_root.join(
480 "ingest/run/test_metric_comp.data/"
481 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
482 ),
483 ),
484 )
485 ),
486 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
487 ),
488 AstropyTable(
489 array(
490 (
491 (
492 "test_metric_comp.output",
493 "foo",
494 "DummyCamComp",
495 "424",
496 "R",
497 "d-r",
498 datastore_root.join(
499 "foo/test_metric_comp.output/"
500 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
501 ),
502 ),
503 (
504 "test_metric_comp.output",
505 "ingest/run",
506 "DummyCamComp",
507 "423",
508 "R",
509 "d-r",
510 datastore_root.join(
511 "ingest/run/test_metric_comp.output/"
512 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
513 ),
514 ),
515 (
516 "test_metric_comp.output",
517 "ingest/run",
518 "DummyCamComp",
519 "424",
520 "R",
521 "d-r",
522 datastore_root.join(
523 "ingest/run/test_metric_comp.output/"
524 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
525 ),
526 ),
527 )
528 ),
529 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
530 ),
531 AstropyTable(
532 array(
533 (
534 (
535 "test_metric_comp.summary",
536 "foo",
537 "DummyCamComp",
538 "424",
539 "R",
540 "d-r",
541 datastore_root.join(
542 "foo/test_metric_comp.summary/"
543 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
544 ),
545 ),
546 (
547 "test_metric_comp.summary",
548 "ingest/run",
549 "DummyCamComp",
550 "423",
551 "R",
552 "d-r",
553 datastore_root.join(
554 "ingest/run/test_metric_comp.summary/"
555 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
556 ),
557 ),
558 (
559 "test_metric_comp.summary",
560 "ingest/run",
561 "DummyCamComp",
562 "424",
563 "R",
564 "d-r",
565 datastore_root.join(
566 "ingest/run/test_metric_comp.summary/"
567 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
568 ),
569 ),
570 )
571 ),
572 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
573 ),
574 )
576 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
578 # Verify that with find first the duplicate dataset is eliminated and
579 # the more recent dataset is returned.
580 tables = self._queryDatasets(
581 repo=testRepo.butler, collections=["chain"], show_uri=True, find_first=True, glob="*"
582 )
584 expectedTables = (
585 AstropyTable(
586 array(
587 (
588 (
589 "test_metric_comp.data",
590 "foo",
591 "DummyCamComp",
592 "424",
593 "R",
594 "d-r",
595 datastore_root.join(
596 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
597 ),
598 ),
599 (
600 "test_metric_comp.data",
601 "ingest/run",
602 "DummyCamComp",
603 "423",
604 "R",
605 "d-r",
606 datastore_root.join(
607 "ingest/run/test_metric_comp.data/"
608 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
609 ),
610 ),
611 )
612 ),
613 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
614 ),
615 AstropyTable(
616 array(
617 (
618 (
619 "test_metric_comp.output",
620 "foo",
621 "DummyCamComp",
622 "424",
623 "R",
624 "d-r",
625 datastore_root.join(
626 "foo/test_metric_comp.output/"
627 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
628 ),
629 ),
630 (
631 "test_metric_comp.output",
632 "ingest/run",
633 "DummyCamComp",
634 "423",
635 "R",
636 "d-r",
637 datastore_root.join(
638 "ingest/run/test_metric_comp.output/"
639 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
640 ),
641 ),
642 )
643 ),
644 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
645 ),
646 AstropyTable(
647 array(
648 (
649 (
650 "test_metric_comp.summary",
651 "foo",
652 "DummyCamComp",
653 "424",
654 "R",
655 "d-r",
656 datastore_root.join(
657 "foo/test_metric_comp.summary/"
658 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
659 ),
660 ),
661 (
662 "test_metric_comp.summary",
663 "ingest/run",
664 "DummyCamComp",
665 "423",
666 "R",
667 "d-r",
668 datastore_root.join(
669 "ingest/run/test_metric_comp.summary/"
670 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
671 ),
672 ),
673 )
674 ),
675 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
676 ),
677 )
679 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
681 # Verify that globs are not supported with find_first=True.
682 with self.assertRaises(InvalidQueryError):
683 self._queryDatasets(
684 repo=testRepo.butler, collections=["*"], show_uri=True, find_first=True, glob="*"
685 )
687 # Collections argument will become mandatory soon
688 with self.assertWarns(FutureWarning):
689 self._queryDatasets(repo=testRepo.butler, glob="*")
692if __name__ == "__main__":
693 unittest.main()