Coverage for tests/test_cliCmdQueryDatasets.py: 42%
60 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-28 04:40 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-28 04:40 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from lsst.resources import ResourcePath
33from numpy import array
35TESTDIR = os.path.abspath(os.path.dirname(__file__))
38def expectedFilesystemDatastoreTables(root: ResourcePath):
39 return (
40 AstropyTable(
41 array(
42 (
43 (
44 "test_metric_comp.data",
45 "ingest/run",
46 "R",
47 "DummyCamComp",
48 "d-r",
49 "423",
50 root.join(
51 "ingest/run/test_metric_comp.data/"
52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
53 ),
54 ),
55 (
56 "test_metric_comp.data",
57 "ingest/run",
58 "R",
59 "DummyCamComp",
60 "d-r",
61 "424",
62 root.join(
63 "ingest/run/test_metric_comp.data/"
64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
65 ),
66 ),
67 )
68 ),
69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
70 ),
71 AstropyTable(
72 array(
73 (
74 (
75 "test_metric_comp.output",
76 "ingest/run",
77 "R",
78 "DummyCamComp",
79 "d-r",
80 "423",
81 root.join(
82 "ingest/run/test_metric_comp.output/"
83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
84 ),
85 ),
86 (
87 "test_metric_comp.output",
88 "ingest/run",
89 "R",
90 "DummyCamComp",
91 "d-r",
92 "424",
93 root.join(
94 "ingest/run/test_metric_comp.output/"
95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
96 ),
97 ),
98 )
99 ),
100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
101 ),
102 AstropyTable(
103 array(
104 (
105 (
106 "test_metric_comp.summary",
107 "ingest/run",
108 "R",
109 "DummyCamComp",
110 "d-r",
111 "423",
112 root.join(
113 "ingest/run/test_metric_comp.summary/"
114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
115 ),
116 ),
117 (
118 "test_metric_comp.summary",
119 "ingest/run",
120 "R",
121 "DummyCamComp",
122 "d-r",
123 "424",
124 root.join(
125 "ingest/run/test_metric_comp.summary/"
126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
127 ),
128 ),
129 )
130 ),
131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
132 ),
133 )
136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
137 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
138 storageClassFactory = StorageClassFactory()
140 @staticmethod
141 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
142 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
144 def setUp(self):
145 self.testdir = makeTestTempDir(TESTDIR)
146 self.repoDir = os.path.join(self.testdir, "repo")
148 def tearDown(self):
149 removeTestTempDir(self.testdir)
151 def testChained(self):
152 testRepo = MetricTestRepo(
153 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
154 )
156 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
158 self.assertAstropyTablesEqual(
159 tables,
160 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root),
161 filterColumns=True,
162 )
164 def testShowURI(self):
165 """Test for expected output with show_uri=True."""
166 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
168 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
170 self.assertAstropyTablesEqual(
171 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True
172 )
174 def testNoShowURI(self):
175 """Test for expected output without show_uri (default is False)."""
176 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
178 tables = self._queryDatasets(repo=self.repoDir)
180 expectedTables = (
181 AstropyTable(
182 array(
183 (
184 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
185 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
186 )
187 ),
188 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
189 ),
190 )
192 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
194 def testWhere(self):
195 """Test using the where clause to reduce the number of rows returned by
196 queryDatasets.
197 """
198 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
200 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
202 expectedTables = (
203 AstropyTable(
204 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")),
205 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
206 ),
207 )
209 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
211 def testGlobDatasetType(self):
212 """Test specifying dataset type."""
213 # Create and register an additional DatasetType
214 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
216 testRepo.butler.registry.insertDimensionData(
217 "visit",
218 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"},
219 )
221 datasetType = addDatasetType(
222 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
223 )
225 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
227 # verify the new dataset type increases the number of tables found:
228 tables = self._queryDatasets(repo=self.repoDir)
230 expectedTables = (
231 AstropyTable(
232 array(
233 (
234 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
235 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
236 )
237 ),
238 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
239 ),
240 AstropyTable(
241 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))),
242 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
243 ),
244 )
246 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
248 def testFindFirstAndCollections(self):
249 """Test the find-first option, and the collections option, since it
250 is required for find-first."""
252 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
254 # Add a new run, and add a dataset to shadow an existing dataset.
255 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
257 # Verify that without find-first, duplicate datasets are returned
258 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
260 expectedTables = (
261 AstropyTable(
262 array(
263 (
264 (
265 "test_metric_comp.data",
266 "foo",
267 "R",
268 "DummyCamComp",
269 "d-r",
270 "424",
271 testRepo.butler.datastore.root.join(
272 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
273 ),
274 ),
275 (
276 "test_metric_comp.data",
277 "ingest/run",
278 "R",
279 "DummyCamComp",
280 "d-r",
281 "423",
282 testRepo.butler.datastore.root.join(
283 "ingest/run/test_metric_comp.data/"
284 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
285 ),
286 ),
287 (
288 "test_metric_comp.data",
289 "ingest/run",
290 "R",
291 "DummyCamComp",
292 "d-r",
293 "424",
294 testRepo.butler.datastore.root.join(
295 "ingest/run/test_metric_comp.data/"
296 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
297 ),
298 ),
299 )
300 ),
301 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
302 ),
303 AstropyTable(
304 array(
305 (
306 (
307 "test_metric_comp.output",
308 "foo",
309 "R",
310 "DummyCamComp",
311 "d-r",
312 "424",
313 testRepo.butler.datastore.root.join(
314 "foo/test_metric_comp.output/"
315 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
316 ),
317 ),
318 (
319 "test_metric_comp.output",
320 "ingest/run",
321 "R",
322 "DummyCamComp",
323 "d-r",
324 "423",
325 testRepo.butler.datastore.root.join(
326 "ingest/run/test_metric_comp.output/"
327 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
328 ),
329 ),
330 (
331 "test_metric_comp.output",
332 "ingest/run",
333 "R",
334 "DummyCamComp",
335 "d-r",
336 "424",
337 testRepo.butler.datastore.root.join(
338 "ingest/run/test_metric_comp.output/"
339 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
340 ),
341 ),
342 )
343 ),
344 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
345 ),
346 AstropyTable(
347 array(
348 (
349 (
350 "test_metric_comp.summary",
351 "foo",
352 "R",
353 "DummyCamComp",
354 "d-r",
355 "424",
356 testRepo.butler.datastore.root.join(
357 "foo/test_metric_comp.summary/"
358 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
359 ),
360 ),
361 (
362 "test_metric_comp.summary",
363 "ingest/run",
364 "R",
365 "DummyCamComp",
366 "d-r",
367 "423",
368 testRepo.butler.datastore.root.join(
369 "ingest/run/test_metric_comp.summary/"
370 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
371 ),
372 ),
373 (
374 "test_metric_comp.summary",
375 "ingest/run",
376 "R",
377 "DummyCamComp",
378 "d-r",
379 "424",
380 testRepo.butler.datastore.root.join(
381 "ingest/run/test_metric_comp.summary/"
382 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
383 ),
384 ),
385 )
386 ),
387 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
388 ),
389 )
391 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
393 # Verify that with find first the duplicate dataset is eliminated and
394 # the more recent dataset is returned.
395 tables = self._queryDatasets(
396 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
397 )
399 expectedTables = (
400 AstropyTable(
401 array(
402 (
403 (
404 "test_metric_comp.data",
405 "foo",
406 "R",
407 "DummyCamComp",
408 "d-r",
409 "424",
410 testRepo.butler.datastore.root.join(
411 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
412 ),
413 ),
414 (
415 "test_metric_comp.data",
416 "ingest/run",
417 "R",
418 "DummyCamComp",
419 "d-r",
420 "423",
421 testRepo.butler.datastore.root.join(
422 "ingest/run/test_metric_comp.data/"
423 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
424 ),
425 ),
426 )
427 ),
428 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
429 ),
430 AstropyTable(
431 array(
432 (
433 (
434 "test_metric_comp.output",
435 "foo",
436 "R",
437 "DummyCamComp",
438 "d-r",
439 "424",
440 testRepo.butler.datastore.root.join(
441 "foo/test_metric_comp.output/"
442 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
443 ),
444 ),
445 (
446 "test_metric_comp.output",
447 "ingest/run",
448 "R",
449 "DummyCamComp",
450 "d-r",
451 "423",
452 testRepo.butler.datastore.root.join(
453 "ingest/run/test_metric_comp.output/"
454 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
455 ),
456 ),
457 )
458 ),
459 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
460 ),
461 AstropyTable(
462 array(
463 (
464 (
465 "test_metric_comp.summary",
466 "foo",
467 "R",
468 "DummyCamComp",
469 "d-r",
470 "424",
471 testRepo.butler.datastore.root.join(
472 "foo/test_metric_comp.summary/"
473 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
474 ),
475 ),
476 (
477 "test_metric_comp.summary",
478 "ingest/run",
479 "R",
480 "DummyCamComp",
481 "d-r",
482 "423",
483 testRepo.butler.datastore.root.join(
484 "ingest/run/test_metric_comp.summary/"
485 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
486 ),
487 ),
488 )
489 ),
490 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
491 ),
492 )
494 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true
498 unittest.main()