Coverage for tests/test_cliCmdQueryDatasets.py: 47%
49 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 02:27 -0700
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 02:27 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-datasets command.
23"""
25import os
26import unittest
28from astropy.table import Table as AstropyTable
29from lsst.daf.butler import StorageClassFactory, script
30from lsst.daf.butler.tests import addDatasetType
31from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir
32from numpy import array
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
39 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
40 storageClassFactory = StorageClassFactory()
42 @staticmethod
43 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
44 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
46 def setUp(self):
47 self.root = makeTestTempDir(TESTDIR)
48 self.testRepo = MetricTestRepo(self.root, configFile=self.configFile)
50 def tearDown(self):
51 removeTestTempDir(self.root)
53 def testShowURI(self):
54 """Test for expected output with show_uri=True."""
55 tables = self._queryDatasets(repo=self.root, show_uri=True)
57 expectedTables = (
58 AstropyTable(
59 array(
60 (
61 (
62 "test_metric_comp.data",
63 "ingest/run",
64 "R",
65 "DummyCamComp",
66 "d-r",
67 "1",
68 "423",
69 self.testRepo.butler.datastore.root.join(
70 "ingest/run/test_metric_comp.data/"
71 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
72 ),
73 ),
74 (
75 "test_metric_comp.data",
76 "ingest/run",
77 "R",
78 "DummyCamComp",
79 "d-r",
80 "1",
81 "424",
82 self.testRepo.butler.datastore.root.join(
83 "ingest/run/test_metric_comp.data/"
84 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
85 ),
86 ),
87 )
88 ),
89 names=(
90 "type",
91 "run",
92 "band",
93 "instrument",
94 "physical_filter",
95 "visit_system",
96 "visit",
97 "URI",
98 ),
99 ),
100 AstropyTable(
101 array(
102 (
103 (
104 "test_metric_comp.output",
105 "ingest/run",
106 "R",
107 "DummyCamComp",
108 "d-r",
109 "1",
110 "423",
111 self.testRepo.butler.datastore.root.join(
112 "ingest/run/test_metric_comp.output/"
113 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
114 ),
115 ),
116 (
117 "test_metric_comp.output",
118 "ingest/run",
119 "R",
120 "DummyCamComp",
121 "d-r",
122 "1",
123 "424",
124 self.testRepo.butler.datastore.root.join(
125 "ingest/run/test_metric_comp.output/"
126 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
127 ),
128 ),
129 )
130 ),
131 names=(
132 "type",
133 "run",
134 "band",
135 "instrument",
136 "physical_filter",
137 "visit_system",
138 "visit",
139 "URI",
140 ),
141 ),
142 AstropyTable(
143 array(
144 (
145 (
146 "test_metric_comp.summary",
147 "ingest/run",
148 "R",
149 "DummyCamComp",
150 "d-r",
151 "1",
152 "423",
153 self.testRepo.butler.datastore.root.join(
154 "ingest/run/test_metric_comp.summary/"
155 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
156 ),
157 ),
158 (
159 "test_metric_comp.summary",
160 "ingest/run",
161 "R",
162 "DummyCamComp",
163 "d-r",
164 "1",
165 "424",
166 self.testRepo.butler.datastore.root.join(
167 "ingest/run/test_metric_comp.summary/"
168 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
169 ),
170 ),
171 )
172 ),
173 names=(
174 "type",
175 "run",
176 "band",
177 "instrument",
178 "physical_filter",
179 "visit_system",
180 "visit",
181 "URI",
182 ),
183 ),
184 )
186 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
188 def testNoShowURI(self):
189 """Test for expected output without show_uri (default is False)."""
190 tables = self._queryDatasets(repo=self.root)
192 expectedTables = (
193 AstropyTable(
194 array(
195 (
196 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "423"),
197 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "424"),
198 )
199 ),
200 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"),
201 ),
202 )
204 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
206 def testWhere(self):
207 """Test using the where clause to reduce the number of rows returned by
208 queryDatasets.
209 """
210 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423")
212 expectedTables = (
213 AstropyTable(
214 array(("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "423")),
215 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"),
216 ),
217 )
219 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
221 def testGlobDatasetType(self):
222 """Test specifying dataset type."""
223 # Create and register an additional DatasetType
225 self.testRepo.butler.registry.insertDimensionData(
226 "visit",
227 {
228 "instrument": "DummyCamComp",
229 "id": 425,
230 "name": "fourtwentyfive",
231 "physical_filter": "d-r",
232 "visit_system": 1,
233 },
234 )
236 datasetType = addDatasetType(
237 self.testRepo.butler,
238 "alt_test_metric_comp",
239 ("instrument", "visit"),
240 "StructuredCompositeReadComp",
241 )
243 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
245 # verify the new dataset type increases the number of tables found:
246 tables = self._queryDatasets(repo=self.root)
248 expectedTables = (
249 AstropyTable(
250 array(
251 (
252 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "423"),
253 ("test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "424"),
254 )
255 ),
256 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"),
257 ),
258 AstropyTable(
259 array((("alt_test_metric_comp", "ingest/run", "R", "DummyCamComp", "d-r", "1", "425"))),
260 names=("type", "run", "band", "instrument", "physical_filter", "visit_system", "visit"),
261 ),
262 )
264 self.assertAstropyTablesEqual(tables, expectedTables, filterColumns=True)
266 def testFindFirstAndCollections(self):
267 """Test the find-first option, and the collections option, since it
268 is required for find-first."""
270 # Add a new run, and add a dataset to shadow an existing dataset.
271 self.testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424})
273 # Verify that without find-first, duplicate datasets are returned
274 tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True)
276 expectedTables = (
277 AstropyTable(
278 array(
279 (
280 (
281 "test_metric_comp.data",
282 "foo",
283 "3",
284 "R",
285 "DummyCamComp",
286 "d-r",
287 "1",
288 "424",
289 self.testRepo.butler.datastore.root.join(
290 "foo/test_metric_comp.data/"
291 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
292 ),
293 ),
294 (
295 "test_metric_comp.data",
296 "ingest/run",
297 "1",
298 "R",
299 "DummyCamComp",
300 "d-r",
301 "1",
302 "423",
303 self.testRepo.butler.datastore.root.join(
304 "ingest/run/test_metric_comp.data/"
305 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
306 ),
307 ),
308 (
309 "test_metric_comp.data",
310 "ingest/run",
311 "2",
312 "R",
313 "DummyCamComp",
314 "d-r",
315 "1",
316 "424",
317 self.testRepo.butler.datastore.root.join(
318 "ingest/run/test_metric_comp.data/"
319 "test_metric_comp_v00000424_fDummyCamComp_data.yaml"
320 ),
321 ),
322 )
323 ),
324 names=(
325 "type",
326 "run",
327 "id",
328 "band",
329 "instrument",
330 "physical_filter",
331 "visit_system",
332 "visit",
333 "URI",
334 ),
335 ),
336 AstropyTable(
337 array(
338 (
339 (
340 "test_metric_comp.output",
341 "foo",
342 "3",
343 "R",
344 "DummyCamComp",
345 "d-r",
346 "1",
347 "424",
348 self.testRepo.butler.datastore.root.join(
349 "foo/test_metric_comp.output/"
350 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
351 ),
352 ),
353 (
354 "test_metric_comp.output",
355 "ingest/run",
356 "1",
357 "R",
358 "DummyCamComp",
359 "d-r",
360 "1",
361 "423",
362 self.testRepo.butler.datastore.root.join(
363 "ingest/run/test_metric_comp.output/"
364 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
365 ),
366 ),
367 (
368 "test_metric_comp.output",
369 "ingest/run",
370 "2",
371 "R",
372 "DummyCamComp",
373 "d-r",
374 "1",
375 "424",
376 self.testRepo.butler.datastore.root.join(
377 "ingest/run/test_metric_comp.output/"
378 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
379 ),
380 ),
381 )
382 ),
383 names=(
384 "type",
385 "run",
386 "id",
387 "band",
388 "instrument",
389 "physical_filter",
390 "visit_system",
391 "visit",
392 "URI",
393 ),
394 ),
395 AstropyTable(
396 array(
397 (
398 (
399 "test_metric_comp.summary",
400 "foo",
401 "3",
402 "R",
403 "DummyCamComp",
404 "d-r",
405 "1",
406 "424",
407 self.testRepo.butler.datastore.root.join(
408 "foo/test_metric_comp.summary/"
409 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
410 ),
411 ),
412 (
413 "test_metric_comp.summary",
414 "ingest/run",
415 "1",
416 "R",
417 "DummyCamComp",
418 "d-r",
419 "1",
420 "423",
421 self.testRepo.butler.datastore.root.join(
422 "ingest/run/test_metric_comp.summary/"
423 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
424 ),
425 ),
426 (
427 "test_metric_comp.summary",
428 "ingest/run",
429 "2",
430 "R",
431 "DummyCamComp",
432 "d-r",
433 "1",
434 "424",
435 self.testRepo.butler.datastore.root.join(
436 "ingest/run/test_metric_comp.summary/"
437 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
438 ),
439 ),
440 )
441 ),
442 names=(
443 "type",
444 "run",
445 "id",
446 "band",
447 "instrument",
448 "physical_filter",
449 "visit_system",
450 "visit",
451 "URI",
452 ),
453 ),
454 )
456 self.assertAstropyTablesEqual(tables, expectedTables)
458 # Verify that with find first the duplicate dataset is eliminated and
459 # the more recent dataset is returned.
460 tables = self._queryDatasets(
461 repo=self.root, collections=["foo", "ingest/run"], show_uri=True, find_first=True
462 )
464 expectedTables = (
465 AstropyTable(
466 array(
467 (
468 (
469 "test_metric_comp.data",
470 "foo",
471 "3",
472 "R",
473 "DummyCamComp",
474 "d-r",
475 "1",
476 "424",
477 self.testRepo.butler.datastore.root.join(
478 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml"
479 ),
480 ),
481 (
482 "test_metric_comp.data",
483 "ingest/run",
484 "1",
485 "R",
486 "DummyCamComp",
487 "d-r",
488 "1",
489 "423",
490 self.testRepo.butler.datastore.root.join(
491 "ingest/run/test_metric_comp.data/"
492 "test_metric_comp_v00000423_fDummyCamComp_data.yaml"
493 ),
494 ),
495 )
496 ),
497 names=(
498 "type",
499 "run",
500 "id",
501 "band",
502 "instrument",
503 "physical_filter",
504 "visit_system",
505 "visit",
506 "URI",
507 ),
508 ),
509 AstropyTable(
510 array(
511 (
512 (
513 "test_metric_comp.output",
514 "foo",
515 "3",
516 "R",
517 "DummyCamComp",
518 "d-r",
519 "1",
520 "424",
521 self.testRepo.butler.datastore.root.join(
522 "foo/test_metric_comp.output/"
523 "test_metric_comp_v00000424_fDummyCamComp_output.yaml"
524 ),
525 ),
526 (
527 "test_metric_comp.output",
528 "ingest/run",
529 "1",
530 "R",
531 "DummyCamComp",
532 "d-r",
533 "1",
534 "423",
535 self.testRepo.butler.datastore.root.join(
536 "ingest/run/test_metric_comp.output/"
537 "test_metric_comp_v00000423_fDummyCamComp_output.yaml"
538 ),
539 ),
540 )
541 ),
542 names=(
543 "type",
544 "run",
545 "id",
546 "band",
547 "instrument",
548 "physical_filter",
549 "visit_system",
550 "visit",
551 "URI",
552 ),
553 ),
554 AstropyTable(
555 array(
556 (
557 (
558 "test_metric_comp.summary",
559 "foo",
560 "3",
561 "R",
562 "DummyCamComp",
563 "d-r",
564 "1",
565 "424",
566 self.testRepo.butler.datastore.root.join(
567 "foo/test_metric_comp.summary/"
568 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml"
569 ),
570 ),
571 (
572 "test_metric_comp.summary",
573 "ingest/run",
574 "1",
575 "R",
576 "DummyCamComp",
577 "d-r",
578 "1",
579 "423",
580 self.testRepo.butler.datastore.root.join(
581 "ingest/run/test_metric_comp.summary/"
582 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml"
583 ),
584 ),
585 )
586 ),
587 names=(
588 "type",
589 "run",
590 "id",
591 "band",
592 "instrument",
593 "physical_filter",
594 "visit_system",
595 "visit",
596 "URI",
597 ),
598 ),
599 )
601 self.assertAstropyTablesEqual(tables, expectedTables)
604if __name__ == "__main__": 604 ↛ 605line 604 didn't jump to line 605, because the condition on line 604 was never true
605 unittest.main()