Coverage for tests/test_cliCmdQueryDatasets.py: 41%
61 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 23:50 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 23:50 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from lsst.resources import ResourcePath
33from numpy import array
35TESTDIR = os.path.abspath(os.path.dirname(__file__))
38def expectedFilesystemDatastoreTables(root: ResourcePath):
39 return (
40 AstropyTable(
41 array(
42 (
43 (
44 "test_metric_comp.data",
45 "ingest/run",
46 "R",
47 "DummyCamComp",
48 "d-r",
49 "423",
50 root.join(
51 "ingest/run/test_metric_comp.data/"
52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
53 ),
54 ),
55 (
56 "test_metric_comp.data",
57 "ingest/run",
58 "R",
59 "DummyCamComp",
60 "d-r",
61 "424",
62 root.join(
63 "ingest/run/test_metric_comp.data/"
64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
65 ),
66 ),
67 )
68 ),
69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
70 ),
71 AstropyTable(
72 array(
73 (
74 (
75 "test_metric_comp.output",
76 "ingest/run",
77 "R",
78 "DummyCamComp",
79 "d-r",
80 "423",
81 root.join(
82 "ingest/run/test_metric_comp.output/"
83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
84 ),
85 ),
86 (
87 "test_metric_comp.output",
88 "ingest/run",
89 "R",
90 "DummyCamComp",
91 "d-r",
92 "424",
93 root.join(
94 "ingest/run/test_metric_comp.output/"
95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
96 ),
97 ),
98 )
99 ),
100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
101 ),
102 AstropyTable(
103 array(
104 (
105 (
106 "test_metric_comp.summary",
107 "ingest/run",
108 "R",
109 "DummyCamComp",
110 "d-r",
111 "423",
112 root.join(
113 "ingest/run/test_metric_comp.summary/"
114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
115 ),
116 ),
117 (
118 "test_metric_comp.summary",
119 "ingest/run",
120 "R",
121 "DummyCamComp",
122 "d-r",
123 "424",
124 root.join(
125 "ingest/run/test_metric_comp.summary/"
126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
127 ),
128 ),
129 )
130 ),
131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
132 ),
133 )
136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
137 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
138 storageClassFactory = StorageClassFactory()
140 @staticmethod
141 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
142 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
144 def setUp(self):
145 self.testdir = makeTestTempDir(TESTDIR)
146 self.repoDir = os.path.join(self.testdir, "repo")
148 def tearDown(self):
149 removeTestTempDir(self.testdir)
151 def testChained(self):
152 root = makeTestTempDir(TESTDIR)
153 testRepo = MetricTestRepo(root, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml"))
155 tables = self._queryDatasets(repo=root, show_uri=True)
157 self.assertAstropyTablesEqual(
158 tables,
159 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root),
160 filterColumns=True,
161 )
163 def testShowURI(self):
164 """Test for expected output with show_uri=True."""
165 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
167 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
169 self.assertAstropyTablesEqual(
170 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True
171 )
173 def testNoShowURI(self):
174 """Test for expected output without show_uri (default is False)."""
175 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
177 tables = self._queryDatasets(repo=self.repoDir)
179 expectedTables = (
180 AstropyTable(
181 array(
182 (
183 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
184 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
185 )
186 ),
187 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
188 ),
189 )
191 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
193 def testWhere(self):
194 """Test using the where clause to reduce the number of rows returned by
195 queryDatasets.
196 """
197 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
199 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
201 expectedTables = (
202 AstropyTable(
203 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")),
204 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
205 ),
206 )
208 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
210 def testGlobDatasetType(self):
211 """Test specifying dataset type."""
212 # Create and register an additional DatasetType
213 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
215 testRepo.butler.registry.insertDimensionData(
216 "visit",
217 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"},
218 )
220 datasetType = addDatasetType(
221 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
222 )
224 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
226 # verify the new dataset type increases the number of tables found:
227 tables = self._queryDatasets(repo=self.repoDir)
229 expectedTables = (
230 AstropyTable(
231 array(
232 (
233 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
234 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
235 )
236 ),
237 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
238 ),
239 AstropyTable(
240 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))),
241 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
242 ),
243 )
245 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
247 def testFindFirstAndCollections(self):
248 """Test the find-first option, and the collections option, since it
249 is required for find-first."""
251 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
253 # Add a new run, and add a dataset to shadow an existing dataset.
254 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
256 # Verify that without find-first, duplicate datasets are returned
257 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
259 expectedTables = (
260 AstropyTable(
261 array(
262 (
263 (
264 "test_metric_comp.data",
265 "foo",
266 "R",
267 "DummyCamComp",
268 "d-r",
269 "424",
270 testRepo.butler.datastore.root.join(
271 "foo/test_metric_comp.data/"
272 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
273 ),
274 ),
275 (
276 "test_metric_comp.data",
277 "ingest/run",
278 "R",
279 "DummyCamComp",
280 "d-r",
281 "423",
282 testRepo.butler.datastore.root.join(
283 "ingest/run/test_metric_comp.data/"
284 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
285 ),
286 ),
287 (
288 "test_metric_comp.data",
289 "ingest/run",
290 "R",
291 "DummyCamComp",
292 "d-r",
293 "424",
294 testRepo.butler.datastore.root.join(
295 "ingest/run/test_metric_comp.data/"
296 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
297 ),
298 ),
299 )
300 ),
301 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
302 ),
303 AstropyTable(
304 array(
305 (
306 (
307 "test_metric_comp.output",
308 "foo",
309 "R",
310 "DummyCamComp",
311 "d-r",
312 "424",
313 testRepo.butler.datastore.root.join(
314 "foo/test_metric_comp.output/"
315 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
316 ),
317 ),
318 (
319 "test_metric_comp.output",
320 "ingest/run",
321 "R",
322 "DummyCamComp",
323 "d-r",
324 "423",
325 testRepo.butler.datastore.root.join(
326 "ingest/run/test_metric_comp.output/"
327 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
328 ),
329 ),
330 (
331 "test_metric_comp.output",
332 "ingest/run",
333 "R",
334 "DummyCamComp",
335 "d-r",
336 "424",
337 testRepo.butler.datastore.root.join(
338 "ingest/run/test_metric_comp.output/"
339 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
340 ),
341 ),
342 )
343 ),
344 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
345 ),
346 AstropyTable(
347 array(
348 (
349 (
350 "test_metric_comp.summary",
351 "foo",
352 "R",
353 "DummyCamComp",
354 "d-r",
355 "424",
356 testRepo.butler.datastore.root.join(
357 "foo/test_metric_comp.summary/"
358 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
359 ),
360 ),
361 (
362 "test_metric_comp.summary",
363 "ingest/run",
364 "R",
365 "DummyCamComp",
366 "d-r",
367 "423",
368 testRepo.butler.datastore.root.join(
369 "ingest/run/test_metric_comp.summary/"
370 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
371 ),
372 ),
373 (
374 "test_metric_comp.summary",
375 "ingest/run",
376 "R",
377 "DummyCamComp",
378 "d-r",
379 "424",
380 testRepo.butler.datastore.root.join(
381 "ingest/run/test_metric_comp.summary/"
382 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
383 ),
384 ),
385 )
386 ),
387 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
388 ),
389 )
391 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
393 # Verify that with find first the duplicate dataset is eliminated and
394 # the more recent dataset is returned.
395 tables = self._queryDatasets(
396 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
397 )
399 expectedTables = (
400 AstropyTable(
401 array(
402 (
403 (
404 "test_metric_comp.data",
405 "foo",
406 "R",
407 "DummyCamComp",
408 "d-r",
409 "424",
410 testRepo.butler.datastore.root.join(
411 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
412 ),
413 ),
414 (
415 "test_metric_comp.data",
416 "ingest/run",
417 "R",
418 "DummyCamComp",
419 "d-r",
420 "423",
421 testRepo.butler.datastore.root.join(
422 "ingest/run/test_metric_comp.data/"
423 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
424 ),
425 ),
426 )
427 ),
428 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
429 ),
430 AstropyTable(
431 array(
432 (
433 (
434 "test_metric_comp.output",
435 "foo",
436 "R",
437 "DummyCamComp",
438 "d-r",
439 "424",
440 testRepo.butler.datastore.root.join(
441 "foo/test_metric_comp.output/"
442 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
443 ),
444 ),
445 (
446 "test_metric_comp.output",
447 "ingest/run",
448 "R",
449 "DummyCamComp",
450 "d-r",
451 "423",
452 testRepo.butler.datastore.root.join(
453 "ingest/run/test_metric_comp.output/"
454 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
455 ),
456 ),
457 )
458 ),
459 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
460 ),
461 AstropyTable(
462 array(
463 (
464 (
465 "test_metric_comp.summary",
466 "foo",
467 "R",
468 "DummyCamComp",
469 "d-r",
470 "424",
471 testRepo.butler.datastore.root.join(
472 "foo/test_metric_comp.summary/"
473 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
474 ),
475 ),
476 (
477 "test_metric_comp.summary",
478 "ingest/run",
479 "R",
480 "DummyCamComp",
481 "d-r",
482 "423",
483 testRepo.butler.datastore.root.join(
484 "ingest/run/test_metric_comp.summary/"
485 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
486 ),
487 ),
488 )
489 ),
490 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
491 ),
492 )
494 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true
498 unittest.main()