Coverage for tests/test_cliCmdQueryDatasets.py: 39%
65 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 10:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Unit tests for daf_butler CLI query-datasets command.
29"""
31import os
32import unittest
34from astropy.table import Table as AstropyTable
35from lsst.daf.butler import StorageClassFactory, script
36from lsst.daf.butler.tests import addDatasetType
37from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
38from lsst.resources import ResourcePath
39from numpy import array
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44def expectedFilesystemDatastoreTables(root: ResourcePath):
45 """Return the expected table contents."""
46 return (
47 AstropyTable(
48 array(
49 (
50 (
51 "test_metric_comp.data",
52 "ingest/run",
53 "DummyCamComp",
54 "423",
55 "R",
56 "d-r",
57 root.join(
58 "ingest/run/test_metric_comp.data/"
59 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
60 ),
61 ),
62 (
63 "test_metric_comp.data",
64 "ingest/run",
65 "DummyCamComp",
66 "424",
67 "R",
68 "d-r",
69 root.join(
70 "ingest/run/test_metric_comp.data/"
71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
72 ),
73 ),
74 )
75 ),
76 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
77 ),
78 AstropyTable(
79 array(
80 (
81 (
82 "test_metric_comp.output",
83 "ingest/run",
84 "DummyCamComp",
85 "423",
86 "R",
87 "d-r",
88 root.join(
89 "ingest/run/test_metric_comp.output/"
90 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
91 ),
92 ),
93 (
94 "test_metric_comp.output",
95 "ingest/run",
96 "DummyCamComp",
97 "424",
98 "R",
99 "d-r",
100 root.join(
101 "ingest/run/test_metric_comp.output/"
102 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
103 ),
104 ),
105 )
106 ),
107 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
108 ),
109 AstropyTable(
110 array(
111 (
112 (
113 "test_metric_comp.summary",
114 "ingest/run",
115 "DummyCamComp",
116 "423",
117 "R",
118 "d-r",
119 root.join(
120 "ingest/run/test_metric_comp.summary/"
121 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
122 ),
123 ),
124 (
125 "test_metric_comp.summary",
126 "ingest/run",
127 "DummyCamComp",
128 "424",
129 "R",
130 "d-r",
131 root.join(
132 "ingest/run/test_metric_comp.summary/"
133 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
134 ),
135 ),
136 )
137 ),
138 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
139 ),
140 )
143class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
144 """Test the query-datasets command-line."""
146 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
147 storageClassFactory = StorageClassFactory()
149 @staticmethod
150 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
151 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
153 def setUp(self):
154 self.testdir = makeTestTempDir(TESTDIR)
155 self.repoDir = os.path.join(self.testdir, "repo")
157 def tearDown(self):
158 removeTestTempDir(self.testdir)
160 def testChained(self):
161 testRepo = MetricTestRepo(
162 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
163 )
165 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
167 # Want second datastore root.
168 roots = testRepo.butler.get_datastore_roots()
169 datastore_root = roots[testRepo.butler.get_datastore_names()[1]]
171 self.assertAstropyTablesEqual(
172 tables,
173 expectedFilesystemDatastoreTables(datastore_root),
174 filterColumns=True,
175 )
177 def testShowURI(self):
178 """Test for expected output with show_uri=True."""
179 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
181 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
183 roots = testRepo.butler.get_datastore_roots()
184 datastore_root = list(roots.values())[0]
186 self.assertAstropyTablesEqual(
187 tables, expectedFilesystemDatastoreTables(datastore_root), filterColumns=True
188 )
190 def testNoShowURI(self):
191 """Test for expected output without show_uri (default is False)."""
192 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
194 tables = self._queryDatasets(repo=self.repoDir)
196 expectedTables = (
197 AstropyTable(
198 array(
199 (
200 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
201 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
202 )
203 ),
204 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
205 ),
206 )
208 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
210 def testWhere(self):
211 """Test using the where clause to reduce the number of rows returned by
212 queryDatasets.
213 """
214 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
216 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
218 expectedTables = (
219 AstropyTable(
220 array(("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r")),
221 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
222 ),
223 )
225 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
227 def testGlobDatasetType(self):
228 """Test specifying dataset type."""
229 # Create and register an additional DatasetType
230 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
232 testRepo.butler.registry.insertDimensionData(
233 "visit",
234 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"},
235 )
237 datasetType = addDatasetType(
238 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
239 )
241 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
243 # verify the new dataset type increases the number of tables found:
244 tables = self._queryDatasets(repo=self.repoDir)
246 expectedTables = (
247 AstropyTable(
248 array(
249 (
250 ("test_metric_comp", "ingest/run", "DummyCamComp", "423", "R", "d-r"),
251 ("test_metric_comp", "ingest/run", "DummyCamComp", "424", "R", "d-r"),
252 )
253 ),
254 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
255 ),
256 AstropyTable(
257 array(("alt_test_metric_comp", "ingest/run", "DummyCamComp", "425", "R", "d-r")),
258 names=("type", "run", "instrument", "visit", "band", "physical_filter"),
259 ),
260 )
262 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
264 def testFindFirstAndCollections(self):
265 """Test the find-first option, and the collections option, since it
266 is required for find-first.
267 """
268 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
270 # Add a new run, and add a dataset to shadow an existing dataset.
271 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
273 # Verify that without find-first, duplicate datasets are returned
274 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
276 # The test should be running with a single FileDatastore.
277 roots = testRepo.butler.get_datastore_roots()
278 assert len(roots) == 1
279 datastore_root = list(roots.values())[0]
281 expectedTables = (
282 AstropyTable(
283 array(
284 (
285 (
286 "test_metric_comp.data",
287 "foo",
288 "DummyCamComp",
289 "424",
290 "R",
291 "d-r",
292 datastore_root.join(
293 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
294 ),
295 ),
296 (
297 "test_metric_comp.data",
298 "ingest/run",
299 "DummyCamComp",
300 "423",
301 "R",
302 "d-r",
303 datastore_root.join(
304 "ingest/run/test_metric_comp.data/"
305 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
306 ),
307 ),
308 (
309 "test_metric_comp.data",
310 "ingest/run",
311 "DummyCamComp",
312 "424",
313 "R",
314 "d-r",
315 datastore_root.join(
316 "ingest/run/test_metric_comp.data/"
317 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
318 ),
319 ),
320 )
321 ),
322 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
323 ),
324 AstropyTable(
325 array(
326 (
327 (
328 "test_metric_comp.output",
329 "foo",
330 "DummyCamComp",
331 "424",
332 "R",
333 "d-r",
334 datastore_root.join(
335 "foo/test_metric_comp.output/"
336 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
337 ),
338 ),
339 (
340 "test_metric_comp.output",
341 "ingest/run",
342 "DummyCamComp",
343 "423",
344 "R",
345 "d-r",
346 datastore_root.join(
347 "ingest/run/test_metric_comp.output/"
348 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
349 ),
350 ),
351 (
352 "test_metric_comp.output",
353 "ingest/run",
354 "DummyCamComp",
355 "424",
356 "R",
357 "d-r",
358 datastore_root.join(
359 "ingest/run/test_metric_comp.output/"
360 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
361 ),
362 ),
363 )
364 ),
365 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
366 ),
367 AstropyTable(
368 array(
369 (
370 (
371 "test_metric_comp.summary",
372 "foo",
373 "DummyCamComp",
374 "424",
375 "R",
376 "d-r",
377 datastore_root.join(
378 "foo/test_metric_comp.summary/"
379 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
380 ),
381 ),
382 (
383 "test_metric_comp.summary",
384 "ingest/run",
385 "DummyCamComp",
386 "423",
387 "R",
388 "d-r",
389 datastore_root.join(
390 "ingest/run/test_metric_comp.summary/"
391 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
392 ),
393 ),
394 (
395 "test_metric_comp.summary",
396 "ingest/run",
397 "DummyCamComp",
398 "424",
399 "R",
400 "d-r",
401 datastore_root.join(
402 "ingest/run/test_metric_comp.summary/"
403 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
404 ),
405 ),
406 )
407 ),
408 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
409 ),
410 )
412 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
414 # Verify that with find first the duplicate dataset is eliminated and
415 # the more recent dataset is returned.
416 tables = self._queryDatasets(
417 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
418 )
420 expectedTables = (
421 AstropyTable(
422 array(
423 (
424 (
425 "test_metric_comp.data",
426 "foo",
427 "DummyCamComp",
428 "424",
429 "R",
430 "d-r",
431 datastore_root.join(
432 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
433 ),
434 ),
435 (
436 "test_metric_comp.data",
437 "ingest/run",
438 "DummyCamComp",
439 "423",
440 "R",
441 "d-r",
442 datastore_root.join(
443 "ingest/run/test_metric_comp.data/"
444 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
445 ),
446 ),
447 )
448 ),
449 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
450 ),
451 AstropyTable(
452 array(
453 (
454 (
455 "test_metric_comp.output",
456 "foo",
457 "DummyCamComp",
458 "424",
459 "R",
460 "d-r",
461 datastore_root.join(
462 "foo/test_metric_comp.output/"
463 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
464 ),
465 ),
466 (
467 "test_metric_comp.output",
468 "ingest/run",
469 "DummyCamComp",
470 "423",
471 "R",
472 "d-r",
473 datastore_root.join(
474 "ingest/run/test_metric_comp.output/"
475 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
476 ),
477 ),
478 )
479 ),
480 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
481 ),
482 AstropyTable(
483 array(
484 (
485 (
486 "test_metric_comp.summary",
487 "foo",
488 "DummyCamComp",
489 "424",
490 "R",
491 "d-r",
492 datastore_root.join(
493 "foo/test_metric_comp.summary/"
494 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
495 ),
496 ),
497 (
498 "test_metric_comp.summary",
499 "ingest/run",
500 "DummyCamComp",
501 "423",
502 "R",
503 "d-r",
504 datastore_root.join(
505 "ingest/run/test_metric_comp.summary/"
506 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
507 ),
508 ),
509 )
510 ),
511 names=("type", "run", "instrument", "visit", "band", "physical_filter", "URI"),
512 ),
513 )
515 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
518if __name__ == "__main__":
519 unittest.main()