Coverage for tests/test_cliCmdQueryDatasets.py: 39%
65 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:53 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:53 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Unit tests for daf_butler CLI query-datasets command.
29"""
31import os
32import unittest
34from astropy.table import Table as AstropyTable
35from lsst.daf.butler import StorageClassFactory, script
36from lsst.daf.butler.tests import addDatasetType
37from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
38from lsst.resources import ResourcePath
39from numpy import array
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44def expectedFilesystemDatastoreTables(root: ResourcePath):
45 """Return the expected table contents."""
46 return (
47 AstropyTable(
48 array(
49 (
50 (
51 "test_metric_comp.data",
52 "ingest/run",
53 "DummyCamComp",
54 "423",
55 "R",
56 "d-r",
57 root.join(
58 "ingest/run/test_metric_comp.data/"
59 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
60 ),
61 ),
62 (
63 "test_metric_comp.data",
64 "ingest/run",
65 "DummyCamComp",
66 "424",
67 "R",
68 "d-r",
69 root.join(
70 "ingest/run/test_metric_comp.data/"
71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
72 ),
73 ),
74 )
75 ),
76 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
77 ),
78 AstropyTable(
79 array(
80 (
81 (
82 "test_metric_comp.output",
83 "ingest/run",
84 "DummyCamComp",
85 "423",
86 "R",
87 "d-r",
88 root.join(
89 "ingest/run/test_metric_comp.output/"
90 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
91 ),
92 ),
93 (
94 "test_metric_comp.output",
95 "ingest/run",
96 "DummyCamComp",
97 "424",
98 "R",
99 "d-r",
100 root.join(
101 "ingest/run/test_metric_comp.output/"
102 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
103 ),
104 ),
105 )
106 ),
107 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
108 ),
109 AstropyTable(
110 array(
111 (
112 (
113 "test_metric_comp.summary",
114 "ingest/run",
115 "DummyCamComp",
116 "423",
117 "R",
118 "d-r",
119 root.join(
120 "ingest/run/test_metric_comp.summary/"
121 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
122 ),
123 ),
124 (
125 "test_metric_comp.summary",
126 "ingest/run",
127 "DummyCamComp",
128 "424",
129 "R",
130 "d-r",
131 root.join(
132 "ingest/run/test_metric_comp.summary/"
133 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
134 ),
135 ),
136 )
137 ),
138 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
139 ),
140 )
143class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
144 """Test the query-datasets command-line."""
146 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
147 storageClassFactory = StorageClassFactory()
149 @staticmethod
150 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
151 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
153 def setUp(self):
154 self.testdir = makeTestTempDir(TESTDIR)
155 self.repoDir = os.path.join(self.testdir, "repo")
157 def tearDown(self):
158 removeTestTempDir(self.testdir)
160 def testChained(self):
161 testRepo = MetricTestRepo(
162 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
163 )
165 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
167 # Want second datastore root.
168 roots = testRepo.butler.get_datastore_roots()
169 datastore_root = roots[testRepo.butler.get_datastore_names()[1]]
171 self.assertAstropyTablesEqual(
172 tables,
173 expectedFilesystemDatastoreTables(datastore_root),
174 filterColumns=True,
175 )
177 def testShowURI(self):
178 """Test for expected output with show_uri=True."""
179 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
181 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
183 roots = testRepo.butler.get_datastore_roots()
184 datastore_root = list(roots.values())[0]
186 self.assertAstropyTablesEqual(
187 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True
188 )
190 def testNoShowURI(self):
191 """Test for expected output without show_uri (default is False)."""
192 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
194 tables = self._queryDatasets(repo=self.repoDir)
196 expectedTables = (
197 AstropyTable(
198 array(
199 (
200 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
201 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
202 )
203 ),
204 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
205 ),
206 )
208 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
210 def testWhere(self):
211 """Test using the where clause to reduce the number of rows returned by
212 queryDatasets.
213 """
214 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
216 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
218 expectedTables = (
219 AstropyTable(
220 array(("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r")),
221 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
222 ),
223 )
225 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
227 def testGlobDatasetType(self):
228 """Test specifying dataset type."""
229 # Create and register an additional DatasetType
230 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
232 testRepo.butler.registry.insertDimensionData(
233 "visit",
234 {
235 "instrument": "DummyCamComp",
236 "id": 425,
237 "name": "fourtwentyfive",
238 "physical_filter": "d-r",
239 "day_obs": 20200101,
240 },
241 )
243 datasetType = addDatasetType(
244 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
245 )
247 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
249 # verify the new dataset type increases the number of tables found:
250 tables = self._queryDatasets(repo=self.repoDir)
252 expectedTables = (
253 AstropyTable(
254 array(
255 (
256 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
257 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
258 )
259 ),
260 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
261 ),
262 AstropyTable(
263 array(("alt_test_metric_comp", "ingest/run", "DummyCamComp", "425", "R", "d-r")),
264 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
265 ),
266 )
268 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
270 def testFindFirstAndCollections(self):
271 """Test the find-first option, and the collections option, since it
272 is required for find-first.
273 """
274 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
276 # Add a new run, and add a dataset to shadow an existing dataset.
277 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
279 # Verify that without find-first, duplicate datasets are returned
280 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
282 # The test should be running with a single FileDatastore.
283 roots = testRepo.butler.get_datastore_roots()
284 assert len(roots) == 1
285 datastore_root = list(roots.values())[0]
287 expectedTables = (
288 AstropyTable(
289 array(
290 (
291 (
292 "test_metric_comp.data",
293 "foo",
294 "DummyCamComp",
295 "424",
296 "R",
297 "d-r",
298 datastore_root.join(
299 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
300 ),
301 ),
302 (
303 "test_metric_comp.data",
304 "ingest/run",
305 "DummyCamComp",
306 "423",
307 "R",
308 "d-r",
309 datastore_root.join(
310 "ingest/run/test_metric_comp.data/"
311 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
312 ),
313 ),
314 (
315 "test_metric_comp.data",
316 "ingest/run",
317 "DummyCamComp",
318 "424",
319 "R",
320 "d-r",
321 datastore_root.join(
322 "ingest/run/test_metric_comp.data/"
323 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
324 ),
325 ),
326 )
327 ),
328 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
329 ),
330 AstropyTable(
331 array(
332 (
333 (
334 "test_metric_comp.output",
335 "foo",
336 "DummyCamComp",
337 "424",
338 "R",
339 "d-r",
340 datastore_root.join(
341 "foo/test_metric_comp.output/"
342 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
343 ),
344 ),
345 (
346 "test_metric_comp.output",
347 "ingest/run",
348 "DummyCamComp",
349 "423",
350 "R",
351 "d-r",
352 datastore_root.join(
353 "ingest/run/test_metric_comp.output/"
354 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
355 ),
356 ),
357 (
358 "test_metric_comp.output",
359 "ingest/run",
360 "DummyCamComp",
361 "424",
362 "R",
363 "d-r",
364 datastore_root.join(
365 "ingest/run/test_metric_comp.output/"
366 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
367 ),
368 ),
369 )
370 ),
371 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
372 ),
373 AstropyTable(
374 array(
375 (
376 (
377 "test_metric_comp.summary",
378 "foo",
379 "DummyCamComp",
380 "424",
381 "R",
382 "d-r",
383 datastore_root.join(
384 "foo/test_metric_comp.summary/"
385 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
386 ),
387 ),
388 (
389 "test_metric_comp.summary",
390 "ingest/run",
391 "DummyCamComp",
392 "423",
393 "R",
394 "d-r",
395 datastore_root.join(
396 "ingest/run/test_metric_comp.summary/"
397 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
398 ),
399 ),
400 (
401 "test_metric_comp.summary",
402 "ingest/run",
403 "DummyCamComp",
404 "424",
405 "R",
406 "d-r",
407 datastore_root.join(
408 "ingest/run/test_metric_comp.summary/"
409 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
410 ),
411 ),
412 )
413 ),
414 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
415 ),
416 )
418 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
420 # Verify that with find first the duplicate dataset is eliminated and
421 # the more recent dataset is returned.
422 tables = self._queryDatasets(
423 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
424 )
426 expectedTables = (
427 AstropyTable(
428 array(
429 (
430 (
431 "test_metric_comp.data",
432 "foo",
433 "DummyCamComp",
434 "424",
435 "R",
436 "d-r",
437 datastore_root.join(
438 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
439 ),
440 ),
441 (
442 "test_metric_comp.data",
443 "ingest/run",
444 "DummyCamComp",
445 "423",
446 "R",
447 "d-r",
448 datastore_root.join(
449 "ingest/run/test_metric_comp.data/"
450 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
451 ),
452 ),
453 )
454 ),
455 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
456 ),
457 AstropyTable(
458 array(
459 (
460 (
461 "test_metric_comp.output",
462 "foo",
463 "DummyCamComp",
464 "424",
465 "R",
466 "d-r",
467 datastore_root.join(
468 "foo/test_metric_comp.output/"
469 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
470 ),
471 ),
472 (
473 "test_metric_comp.output",
474 "ingest/run",
475 "DummyCamComp",
476 "423",
477 "R",
478 "d-r",
479 datastore_root.join(
480 "ingest/run/test_metric_comp.output/"
481 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
482 ),
483 ),
484 )
485 ),
486 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
487 ),
488 AstropyTable(
489 array(
490 (
491 (
492 "test_metric_comp.summary",
493 "foo",
494 "DummyCamComp",
495 "424",
496 "R",
497 "d-r",
498 datastore_root.join(
499 "foo/test_metric_comp.summary/"
500 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
501 ),
502 ),
503 (
504 "test_metric_comp.summary",
505 "ingest/run",
506 "DummyCamComp",
507 "423",
508 "R",
509 "d-r",
510 datastore_root.join(
511 "ingest/run/test_metric_comp.summary/"
512 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
513 ),
514 ),
515 )
516 ),
517 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
518 ),
519 )
521 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
524if __name__ == "__main__":
525 unittest.main()