Coverage for tests/test_cliCmdQueryDatasets.py : 47%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-collections command.
23"""
25from astropy.table import Table as AstropyTable
26from numpy import array
27import os
28import unittest
30from lsst.daf.butler import StorageClassFactory
31from lsst.daf.butler import script
32from lsst.daf.butler.tests import addDatasetType
33from lsst.daf.butler.tests.utils import ButlerTestHelper, makeTestTempDir, MetricTestRepo, removeTestTempDir
36TESTDIR = os.path.abspath(os.path.dirname(__file__))
39class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
41 mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDatasets"
43 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
44 storageClassFactory = StorageClassFactory()
46 @staticmethod
47 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
48 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri)
50 def setUp(self):
51 self.root = makeTestTempDir(TESTDIR)
52 self.testRepo = MetricTestRepo(self.root,
53 configFile=os.path.join(TESTDIR, "config/basic/butler.yaml"))
55 def tearDown(self):
56 removeTestTempDir(self.root)
58 def testShowURI(self):
59 """Test for expected output with show_uri=True."""
60 tables = self._queryDatasets(repo=self.root, show_uri=True)
62 expectedTables = (
63 AstropyTable(array((
64 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
65 self.testRepo.butler.datastore.root.join(
66 "ingest/run/test_metric_comp.data/"
67 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
68 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
69 self.testRepo.butler.datastore.root.join(
70 "ingest/run/test_metric_comp.data/"
71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))),
72 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
73 "visit", "URI")),
74 AstropyTable(array((
75 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
76 self.testRepo.butler.datastore.root.join(
77 "ingest/run/test_metric_comp.output/"
78 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
79 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
80 self.testRepo.butler.datastore.root.join(
81 "ingest/run/test_metric_comp.output/"
82 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))),
83 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
84 "visit", "URI")),
85 AstropyTable(array((
86 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
87 self.testRepo.butler.datastore.root.join(
88 "ingest/run/test_metric_comp.summary/"
89 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
90 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
91 self.testRepo.butler.datastore.root.join(
92 "ingest/run/test_metric_comp.summary/"
93 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))),
94 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
95 "visit", "URI")),
96 )
98 self.assertAstropyTablesEqual(tables, expectedTables)
100 def testNoShowURI(self):
101 """Test for expected output without show_uri (default is False)."""
102 tables = self._queryDatasets(repo=self.root)
104 expectedTables = (
105 AstropyTable(array((
106 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
107 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
108 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
109 ),
110 )
112 self.assertAstropyTablesEqual(tables, expectedTables)
114 def testWhere(self):
115 """Test using the where clause to reduce the number of rows returned.
116 """
117 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423")
119 expectedTables = (
120 AstropyTable(array(
121 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")),
122 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"),
123 ),
124 )
126 self.assertAstropyTablesEqual(tables, expectedTables)
128 def testGlobDatasetType(self):
129 """Test specifying dataset type."""
130 # Create and register an additional DatasetType
132 self.testRepo.butler.registry.insertDimensionData("visit",
133 {"instrument": "DummyCamComp", "id": 425,
134 "name": "fourtwentyfive", "physical_filter": "d-r",
135 "visit_system": 1})
137 datasetType = addDatasetType(self.testRepo.butler,
138 "alt_test_metric_comp",
139 ("instrument", "visit"),
140 "StructuredCompositeReadComp")
142 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
144 # verify the new dataset type increases the number of tables found:
145 tables = self._queryDatasets(repo=self.root)
147 expectedTables = (
148 AstropyTable(array((
149 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
150 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
151 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
152 ),
153 AstropyTable(array((
154 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))),
155 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
156 )
157 )
159 self.assertAstropyTablesEqual(tables, expectedTables)
161 def testFindFirstAndCollections(self):
162 """Test the find-first option, and the collections option, since it
163 is required for find-first."""
165 # Add a new run, and add a dataset to shadow an existing dataset.
166 self.testRepo.addDataset(run="foo",
167 dataId={"instrument": "DummyCamComp", "visit": 424})
169 # Verify that without find-first, duplicate datasets are returned
170 tables = self._queryDatasets(repo=self.root,
171 collections=["foo", "ingest/run"],
172 show_uri=True)
174 expectedTables = (
175 AstropyTable(array(
176 (
177 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
178 self.testRepo.butler.datastore.root.join(
179 "foo/test_metric_comp.data/"
180 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
181 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
182 self.testRepo.butler.datastore.root.join(
183 "ingest/run/test_metric_comp.data/"
184 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
185 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
186 self.testRepo.butler.datastore.root.join(
187 "ingest/run/test_metric_comp.data/"
188 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
189 )),
190 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
191 "visit", "URI")),
192 AstropyTable(array(
193 (
194 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
195 self.testRepo.butler.datastore.root.join(
196 "foo/test_metric_comp.output/"
197 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
198 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
199 self.testRepo.butler.datastore.root.join(
200 "ingest/run/test_metric_comp.output/"
201 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
202 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
203 self.testRepo.butler.datastore.root.join(
204 "ingest/run/test_metric_comp.output/"
205 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
206 )),
207 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
208 "visit", "URI")),
209 AstropyTable(array(
210 (
211 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
212 self.testRepo.butler.datastore.root.join(
213 "foo/test_metric_comp.summary/"
214 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
215 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
216 self.testRepo.butler.datastore.root.join(
217 "ingest/run/test_metric_comp.summary/"
218 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
219 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
220 self.testRepo.butler.datastore.root.join(
221 "ingest/run/test_metric_comp.summary/"
222 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
223 )),
224 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
225 "visit", "URI")),
226 )
228 self.assertAstropyTablesEqual(tables, expectedTables)
230 # Verify that with find first the duplicate dataset is eliminated and
231 # the more recent dataset is returned.
232 tables = self._queryDatasets(repo=self.root,
233 collections=["foo", "ingest/run"],
234 show_uri=True,
235 find_first=True)
237 expectedTables = (
238 AstropyTable(array(
239 (
240 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
241 self.testRepo.butler.datastore.root.join(
242 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
243 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
244 self.testRepo.butler.datastore.root.join(
245 "ingest/run/test_metric_comp.data/"
246 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
247 )),
248 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
249 "visit", "URI")),
250 AstropyTable(array(
251 (
252 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
253 self.testRepo.butler.datastore.root.join(
254 "foo/test_metric_comp.output/"
255 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
256 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
257 self.testRepo.butler.datastore.root.join(
258 "ingest/run/test_metric_comp.output/"
259 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
260 )),
261 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
262 "visit", "URI")),
263 AstropyTable(array(
264 (
265 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
266 self.testRepo.butler.datastore.root.join(
267 "foo/test_metric_comp.summary/"
268 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
269 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
270 self.testRepo.butler.datastore.root.join(
271 "ingest/run/test_metric_comp.summary/"
272 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
273 )),
274 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
275 "visit", "URI")),
276 )
278 self.assertAstropyTablesEqual(tables, expectedTables)
281if __name__ == "__main__": 281 ↛ 282line 281 didn't jump to line 282, because the condition on line 281 was never true
282 unittest.main()