Coverage for tests/test_cliCmdQueryDatasets.py: 42%
60 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-15 01:59 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-15 01:59 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from lsst.resources import ResourcePath
33from numpy import array
35TESTDIR = os.path.abspath(os.path.dirname(__file__))
38def expectedFilesystemDatastoreTables(root: ResourcePath):
39 return (
40 AstropyTable(
41 array(
42 (
43 (
44 "test_metric_comp.data",
45 "ingest/run",
46 "R",
47 "DummyCamComp",
48 "d-r",
49 "423",
50 root.join(
51 "ingest/run/test_metric_comp.data/"
52 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
53 ),
54 ),
55 (
56 "test_metric_comp.data",
57 "ingest/run",
58 "R",
59 "DummyCamComp",
60 "d-r",
61 "424",
62 root.join(
63 "ingest/run/test_metric_comp.data/"
64 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
65 ),
66 ),
67 )
68 ),
69 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
70 ),
71 AstropyTable(
72 array(
73 (
74 (
75 "test_metric_comp.output",
76 "ingest/run",
77 "R",
78 "DummyCamComp",
79 "d-r",
80 "423",
81 root.join(
82 "ingest/run/test_metric_comp.output/"
83 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
84 ),
85 ),
86 (
87 "test_metric_comp.output",
88 "ingest/run",
89 "R",
90 "DummyCamComp",
91 "d-r",
92 "424",
93 root.join(
94 "ingest/run/test_metric_comp.output/"
95 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
96 ),
97 ),
98 )
99 ),
100 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
101 ),
102 AstropyTable(
103 array(
104 (
105 (
106 "test_metric_comp.summary",
107 "ingest/run",
108 "R",
109 "DummyCamComp",
110 "d-r",
111 "423",
112 root.join(
113 "ingest/run/test_metric_comp.summary/"
114 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
115 ),
116 ),
117 (
118 "test_metric_comp.summary",
119 "ingest/run",
120 "R",
121 "DummyCamComp",
122 "d-r",
123 "424",
124 root.join(
125 "ingest/run/test_metric_comp.summary/"
126 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
127 ),
128 ),
129 )
130 ),
131 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
132 ),
133 )
136class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
138 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
139 storageClassFactory = StorageClassFactory()
141 @staticmethod
142 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
143 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
145 def setUp(self):
146 self.testdir = makeTestTempDir(TESTDIR)
147 self.repoDir = os.path.join(self.testdir, "repo")
149 def tearDown(self):
150 removeTestTempDir(self.testdir)
152 def testChained(self):
153 testRepo = MetricTestRepo(
154 self.repoDir, configFile=os.path.join(TESTDIR, "config/basic/butler-chained.yaml")
155 )
157 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
159 self.assertAstropyTablesEqual(
160 tables,
161 expectedFilesystemDatastoreTables(testRepo.butler.datastore.datastores[1].root),
162 filterColumns=True,
163 )
165 def testShowURI(self):
166 """Test for expected output with show_uri=True."""
167 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
169 tables = self._queryDatasets(repo=self.repoDir, show_uri=True)
171 self.assertAstropyTablesEqual(
172 tables, expectedFilesystemDatastoreTables(testRepo.butler.datastore.root), filterColumns=True
173 )
175 def testNoShowURI(self):
176 """Test for expected output without show_uri (default is False)."""
177 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
179 tables = self._queryDatasets(repo=self.repoDir)
181 expectedTables = (
182 AstropyTable(
183 array(
184 (
185 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
186 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
187 )
188 ),
189 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
190 ),
191 )
193 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
195 def testWhere(self):
196 """Test using the where clause to reduce the number of rows returned by
197 queryDatasets.
198 """
199 _ = MetricTestRepo(self.repoDir, configFile=self.configFile)
201 tables = self._queryDatasets(repo=self.repoDir, where="instrument='DummyCamComp' AND visit=423")
203 expectedTables = (
204 AstropyTable(
205 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")),
206 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
207 ),
208 )
210 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
212 def testGlobDatasetType(self):
213 """Test specifying dataset type."""
214 # Create and register an additional DatasetType
215 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
217 testRepo.butler.registry.insertDimensionData(
218 "visit",
219 {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r"},
220 )
222 datasetType = addDatasetType(
223 testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp"
224 )
226 testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
228 # verify the new dataset type increases the number of tables found:
229 tables = self._queryDatasets(repo=self.repoDir)
231 expectedTables = (
232 AstropyTable(
233 array(
234 (
235 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
236 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
237 )
238 ),
239 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
240 ),
241 AstropyTable(
242 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))),
243 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
244 ),
245 )
247 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
249 def testFindFirstAndCollections(self):
250 """Test the find-first option, and the collections option, since it
251 is required for find-first."""
253 testRepo = MetricTestRepo(self.repoDir, configFile=self.configFile)
255 # Add a new run, and add a dataset to shadow an existing dataset.
256 testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
258 # Verify that without find-first, duplicate datasets are returned
259 tables = self._queryDatasets(repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True)
261 expectedTables = (
262 AstropyTable(
263 array(
264 (
265 (
266 "test_metric_comp.data",
267 "foo",
268 "R",
269 "DummyCamComp",
270 "d-r",
271 "424",
272 testRepo.butler.datastore.root.join(
273 "foo/test_metric_comp.data/"
274 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
275 ),
276 ),
277 (
278 "test_metric_comp.data",
279 "ingest/run",
280 "R",
281 "DummyCamComp",
282 "d-r",
283 "423",
284 testRepo.butler.datastore.root.join(
285 "ingest/run/test_metric_comp.data/"
286 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
287 ),
288 ),
289 (
290 "test_metric_comp.data",
291 "ingest/run",
292 "R",
293 "DummyCamComp",
294 "d-r",
295 "424",
296 testRepo.butler.datastore.root.join(
297 "ingest/run/test_metric_comp.data/"
298 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
299 ),
300 ),
301 )
302 ),
303 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
304 ),
305 AstropyTable(
306 array(
307 (
308 (
309 "test_metric_comp.output",
310 "foo",
311 "R",
312 "DummyCamComp",
313 "d-r",
314 "424",
315 testRepo.butler.datastore.root.join(
316 "foo/test_metric_comp.output/"
317 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
318 ),
319 ),
320 (
321 "test_metric_comp.output",
322 "ingest/run",
323 "R",
324 "DummyCamComp",
325 "d-r",
326 "423",
327 testRepo.butler.datastore.root.join(
328 "ingest/run/test_metric_comp.output/"
329 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
330 ),
331 ),
332 (
333 "test_metric_comp.output",
334 "ingest/run",
335 "R",
336 "DummyCamComp",
337 "d-r",
338 "424",
339 testRepo.butler.datastore.root.join(
340 "ingest/run/test_metric_comp.output/"
341 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
342 ),
343 ),
344 )
345 ),
346 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
347 ),
348 AstropyTable(
349 array(
350 (
351 (
352 "test_metric_comp.summary",
353 "foo",
354 "R",
355 "DummyCamComp",
356 "d-r",
357 "424",
358 testRepo.butler.datastore.root.join(
359 "foo/test_metric_comp.summary/"
360 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
361 ),
362 ),
363 (
364 "test_metric_comp.summary",
365 "ingest/run",
366 "R",
367 "DummyCamComp",
368 "d-r",
369 "423",
370 testRepo.butler.datastore.root.join(
371 "ingest/run/test_metric_comp.summary/"
372 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
373 ),
374 ),
375 (
376 "test_metric_comp.summary",
377 "ingest/run",
378 "R",
379 "DummyCamComp",
380 "d-r",
381 "424",
382 testRepo.butler.datastore.root.join(
383 "ingest/run/test_metric_comp.summary/"
384 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
385 ),
386 ),
387 )
388 ),
389 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
390 ),
391 )
393 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
395 # Verify that with find first the duplicate dataset is eliminated and
396 # the more recent dataset is returned.
397 tables = self._queryDatasets(
398 repo=self.repoDir, collections=["foo", "ingest/run"], show_uri=True, find_first=True
399 )
401 expectedTables = (
402 AstropyTable(
403 array(
404 (
405 (
406 "test_metric_comp.data",
407 "foo",
408 "R",
409 "DummyCamComp",
410 "d-r",
411 "424",
412 testRepo.butler.datastore.root.join(
413 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
414 ),
415 ),
416 (
417 "test_metric_comp.data",
418 "ingest/run",
419 "R",
420 "DummyCamComp",
421 "d-r",
422 "423",
423 testRepo.butler.datastore.root.join(
424 "ingest/run/test_metric_comp.data/"
425 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
426 ),
427 ),
428 )
429 ),
430 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
431 ),
432 AstropyTable(
433 array(
434 (
435 (
436 "test_metric_comp.output",
437 "foo",
438 "R",
439 "DummyCamComp",
440 "d-r",
441 "424",
442 testRepo.butler.datastore.root.join(
443 "foo/test_metric_comp.output/"
444 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
445 ),
446 ),
447 (
448 "test_metric_comp.output",
449 "ingest/run",
450 "R",
451 "DummyCamComp",
452 "d-r",
453 "423",
454 testRepo.butler.datastore.root.join(
455 "ingest/run/test_metric_comp.output/"
456 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
457 ),
458 ),
459 )
460 ),
461 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
462 ),
463 AstropyTable(
464 array(
465 (
466 (
467 "test_metric_comp.summary",
468 "foo",
469 "R",
470 "DummyCamComp",
471 "d-r",
472 "424",
473 testRepo.butler.datastore.root.join(
474 "foo/test_metric_comp.summary/"
475 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
476 ),
477 ),
478 (
479 "test_metric_comp.summary",
480 "ingest/run",
481 "R",
482 "DummyCamComp",
483 "d-r",
484 "423",
485 testRepo.butler.datastore.root.join(
486 "ingest/run/test_metric_comp.summary/"
487 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
488 ),
489 ),
490 )
491 ),
492 names=("type", "run", "band", "instrument", "physical_filter", "visit", "URI"),
493 ),
494 )
496 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
499if __name__ == "__main__": 499 ↛ 500line 499 didn't jump to line 500, because the condition on line 499 was never true
500 unittest.main()