Coverage for tests/test_cliCmdQueryDatasets.py: 47%
49 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-09 09:43 +0000
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-09 09:43 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from numpy import array
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
39 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
40 storageClassFactory = StorageClassFactory()
42 @staticmethod
43 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
44 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
46 def setUp(self):
47 self.root = makeTestTempDir(TESTDIR)
48 self.testRepo = MetricTestRepo(self.root, configFile=self.configFile)
50 def tearDown(self):
51 removeTestTempDir(self.root)
53 def testShowURI(self):
54 """Test for expected output with show_uri=True."""
55 tables = self._queryDatasets(repo=self.root, show_uri=True)
57 expectedTables = (
58 AstropyTable(
59 array(
60 (
61 (
62 "test_metric_comp.data",
63 "ingest/run",
64 "R",
65 "DummyCamComp",
66 "d-r",
67 "423",
68 self.testRepo.butler.datastore.root.join(
69 "ingest/run/test_metric_comp.data/"
70 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
71 ),
72 ),
73 (
74 "test_metric_comp.data",
75 "ingest/run",
76 "R",
77 "DummyCamComp",
78 "d-r",
79 "424",
80 self.testRepo.butler.datastore.root.join(
81 "ingest/run/test_metric_comp.data/"
82 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
83 ),
84 ),
85 )
86 ),
87 names=(
88 "type",
89 "run",
90 "band",
91 "instrument",
92 "physical_filter",
93 "visit",
94 "URI",
95 ),
96 ),
97 AstropyTable(
98 array(
99 (
100 (
101 "test_metric_comp.output",
102 "ingest/run",
103 "R",
104 "DummyCamComp",
105 "d-r",
106 "423",
107 self.testRepo.butler.datastore.root.join(
108 "ingest/run/test_metric_comp.output/"
109 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
110 ),
111 ),
112 (
113 "test_metric_comp.output",
114 "ingest/run",
115 "R",
116 "DummyCamComp",
117 "d-r",
118 "424",
119 self.testRepo.butler.datastore.root.join(
120 "ingest/run/test_metric_comp.output/"
121 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
122 ),
123 ),
124 )
125 ),
126 names=(
127 "type",
128 "run",
129 "band",
130 "instrument",
131 "physical_filter",
132 "visit",
133 "URI",
134 ),
135 ),
136 AstropyTable(
137 array(
138 (
139 (
140 "test_metric_comp.summary",
141 "ingest/run",
142 "R",
143 "DummyCamComp",
144 "d-r",
145 "423",
146 self.testRepo.butler.datastore.root.join(
147 "ingest/run/test_metric_comp.summary/"
148 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
149 ),
150 ),
151 (
152 "test_metric_comp.summary",
153 "ingest/run",
154 "R",
155 "DummyCamComp",
156 "d-r",
157 "424",
158 self.testRepo.butler.datastore.root.join(
159 "ingest/run/test_metric_comp.summary/"
160 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
161 ),
162 ),
163 )
164 ),
165 names=(
166 "type",
167 "run",
168 "band",
169 "instrument",
170 "physical_filter",
171 "visit",
172 "URI",
173 ),
174 ),
175 )
177 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
179 def testNoShowURI(self):
180 """Test for expected output without show_uri (default is False)."""
181 tables = self._queryDatasets(repo=self.root)
183 expectedTables = (
184 AstropyTable(
185 array(
186 (
187 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
188 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
189 )
190 ),
191 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
192 ),
193 )
195 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
197 def testWhere(self):
198 """Test using the where clause to reduce the number of rows returned by
199 queryDatasets.
200 """
201 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423")
203 expectedTables = (
204 AstropyTable(
205 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423")),
206 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
207 ),
208 )
210 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
212 def testGlobDatasetType(self):
213 """Test specifying dataset type."""
214 # Create and register an additional DatasetType
216 self.testRepo.butler.registry.insertDimensionData(
217 "visit",
218 {
219 "instrument": "DummyCamComp",
220 "id": 425,
221 "name": "fourtwentyfive",
222 "physical_filter": "d-r",
223 },
224 )
226 datasetType = addDatasetType(
227 self.testRepo.butler,
228 "alt_test_metric_comp",
229 ("instrument", "visit"),
230 "StructuredCompositeReadComp",
231 )
233 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
235 # verify the new dataset type increases the number of tables found:
236 tables = self._queryDatasets(repo=self.root)
238 expectedTables = (
239 AstropyTable(
240 array(
241 (
242 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "423"),
243 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "424"),
244 )
245 ),
246 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
247 ),
248 AstropyTable(
249 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "425"))),
250 names=("type", "run", "band", "instrument", "physical_filter", "visit"),
251 ),
252 )
254 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
256 def testFindFirstAndCollections(self):
257 """Test the find-first option, and the collections option, since it
258 is required for find-first."""
260 # Add a new run, and add a dataset to shadow an existing dataset.
261 self.testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
263 # Verify that without find-first, duplicate datasets are returned
264 tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True)
266 expectedTables = (
267 AstropyTable(
268 array(
269 (
270 (
271 "test_metric_comp.data",
272 "foo",
273 "3",
274 "R",
275 "DummyCamComp",
276 "d-r",
277 "424",
278 self.testRepo.butler.datastore.root.join(
279 "foo/test_metric_comp.data/"
280 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
281 ),
282 ),
283 (
284 "test_metric_comp.data",
285 "ingest/run",
286 "1",
287 "R",
288 "DummyCamComp",
289 "d-r",
290 "423",
291 self.testRepo.butler.datastore.root.join(
292 "ingest/run/test_metric_comp.data/"
293 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
294 ),
295 ),
296 (
297 "test_metric_comp.data",
298 "ingest/run",
299 "2",
300 "R",
301 "DummyCamComp",
302 "d-r",
303 "424",
304 self.testRepo.butler.datastore.root.join(
305 "ingest/run/test_metric_comp.data/"
306 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
307 ),
308 ),
309 )
310 ),
311 names=(
312 "type",
313 "run",
314 "id",
315 "band",
316 "instrument",
317 "physical_filter",
318 "visit",
319 "URI",
320 ),
321 ),
322 AstropyTable(
323 array(
324 (
325 (
326 "test_metric_comp.output",
327 "foo",
328 "3",
329 "R",
330 "DummyCamComp",
331 "d-r",
332 "424",
333 self.testRepo.butler.datastore.root.join(
334 "foo/test_metric_comp.output/"
335 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
336 ),
337 ),
338 (
339 "test_metric_comp.output",
340 "ingest/run",
341 "1",
342 "R",
343 "DummyCamComp",
344 "d-r",
345 "423",
346 self.testRepo.butler.datastore.root.join(
347 "ingest/run/test_metric_comp.output/"
348 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
349 ),
350 ),
351 (
352 "test_metric_comp.output",
353 "ingest/run",
354 "2",
355 "R",
356 "DummyCamComp",
357 "d-r",
358 "424",
359 self.testRepo.butler.datastore.root.join(
360 "ingest/run/test_metric_comp.output/"
361 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
362 ),
363 ),
364 )
365 ),
366 names=(
367 "type",
368 "run",
369 "id",
370 "band",
371 "instrument",
372 "physical_filter",
373 "visit",
374 "URI",
375 ),
376 ),
377 AstropyTable(
378 array(
379 (
380 (
381 "test_metric_comp.summary",
382 "foo",
383 "3",
384 "R",
385 "DummyCamComp",
386 "d-r",
387 "424",
388 self.testRepo.butler.datastore.root.join(
389 "foo/test_metric_comp.summary/"
390 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
391 ),
392 ),
393 (
394 "test_metric_comp.summary",
395 "ingest/run",
396 "1",
397 "R",
398 "DummyCamComp",
399 "d-r",
400 "423",
401 self.testRepo.butler.datastore.root.join(
402 "ingest/run/test_metric_comp.summary/"
403 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
404 ),
405 ),
406 (
407 "test_metric_comp.summary",
408 "ingest/run",
409 "2",
410 "R",
411 "DummyCamComp",
412 "d-r",
413 "424",
414 self.testRepo.butler.datastore.root.join(
415 "ingest/run/test_metric_comp.summary/"
416 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
417 ),
418 ),
419 )
420 ),
421 names=(
422 "type",
423 "run",
424 "id",
425 "band",
426 "instrument",
427 "physical_filter",
428 "visit",
429 "URI",
430 ),
431 ),
432 )
434 self.assertAstropyTablesEqual(tables, expectedTables)
436 # Verify that with find first the duplicate dataset is eliminated and
437 # the more recent dataset is returned.
438 tables = self._queryDatasets(
439 repo=self.root, collections=["foo", "ingest/run"], show_uri=True, find_first=True
440 )
442 expectedTables = (
443 AstropyTable(
444 array(
445 (
446 (
447 "test_metric_comp.data",
448 "foo",
449 "3",
450 "R",
451 "DummyCamComp",
452 "d-r",
453 "424",
454 self.testRepo.butler.datastore.root.join(
455 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
456 ),
457 ),
458 (
459 "test_metric_comp.data",
460 "ingest/run",
461 "1",
462 "R",
463 "DummyCamComp",
464 "d-r",
465 "423",
466 self.testRepo.butler.datastore.root.join(
467 "ingest/run/test_metric_comp.data/"
468 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
469 ),
470 ),
471 )
472 ),
473 names=(
474 "type",
475 "run",
476 "id",
477 "band",
478 "instrument",
479 "physical_filter",
480 "visit",
481 "URI",
482 ),
483 ),
484 AstropyTable(
485 array(
486 (
487 (
488 "test_metric_comp.output",
489 "foo",
490 "3",
491 "R",
492 "DummyCamComp",
493 "d-r",
494 "424",
495 self.testRepo.butler.datastore.root.join(
496 "foo/test_metric_comp.output/"
497 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
498 ),
499 ),
500 (
501 "test_metric_comp.output",
502 "ingest/run",
503 "1",
504 "R",
505 "DummyCamComp",
506 "d-r",
507 "423",
508 self.testRepo.butler.datastore.root.join(
509 "ingest/run/test_metric_comp.output/"
510 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
511 ),
512 ),
513 )
514 ),
515 names=(
516 "type",
517 "run",
518 "id",
519 "band",
520 "instrument",
521 "physical_filter",
522 "visit",
523 "URI",
524 ),
525 ),
526 AstropyTable(
527 array(
528 (
529 (
530 "test_metric_comp.summary",
531 "foo",
532 "3",
533 "R",
534 "DummyCamComp",
535 "d-r",
536 "424",
537 self.testRepo.butler.datastore.root.join(
538 "foo/test_metric_comp.summary/"
539 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
540 ),
541 ),
542 (
543 "test_metric_comp.summary",
544 "ingest/run",
545 "1",
546 "R",
547 "DummyCamComp",
548 "d-r",
549 "423",
550 self.testRepo.butler.datastore.root.join(
551 "ingest/run/test_metric_comp.summary/"
552 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
553 ),
554 ),
555 )
556 ),
557 names=(
558 "type",
559 "run",
560 "id",
561 "band",
562 "instrument",
563 "physical_filter",
564 "visit",
565 "URI",
566 ),
567 ),
568 )
570 self.assertAstropyTablesEqual(tables, expectedTables)
573if __name__ == "__main__": 573 ↛ 574line 573 didn't jump to line 574, because the condition on line 573 was never true
574 unittest.main()