Coverage for tests/test_cliCmdQueryDatasets.py : 46%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-collections command.
23"""
25from astropy.table import Table as AstropyTable
26from numpy import array
27import os
28import unittest
30from lsst.daf.butler import StorageClassFactory
31from lsst.daf.butler import script
32from lsst.daf.butler.tests import addDatasetType
33from lsst.daf.butler.tests.utils import ButlerTestHelper, makeTestTempDir, MetricTestRepo, removeTestTempDir
36TESTDIR = os.path.abspath(os.path.dirname(__file__))
39class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
41 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
42 storageClassFactory = StorageClassFactory()
44 @staticmethod
45 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
46 return script.QueryDatasets(glob, collections, where, find_first, show_uri, repo=repo).getTables()
48 def setUp(self):
49 self.root = makeTestTempDir(TESTDIR)
50 self.testRepo = MetricTestRepo(self.root,
51 configFile=os.path.join(TESTDIR, "config/basic/butler.yaml"))
53 def tearDown(self):
54 removeTestTempDir(self.root)
56 def testShowURI(self):
57 """Test for expected output with show_uri=True."""
58 tables = self._queryDatasets(repo=self.root, show_uri=True)
60 expectedTables = (
61 AstropyTable(array((
62 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
63 self.testRepo.butler.datastore.root.join(
64 "ingest/run/test_metric_comp.data/"
65 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
66 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
67 self.testRepo.butler.datastore.root.join(
68 "ingest/run/test_metric_comp.data/"
69 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))),
70 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
71 "visit", "URI")),
72 AstropyTable(array((
73 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
74 self.testRepo.butler.datastore.root.join(
75 "ingest/run/test_metric_comp.output/"
76 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
77 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
78 self.testRepo.butler.datastore.root.join(
79 "ingest/run/test_metric_comp.output/"
80 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))),
81 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
82 "visit", "URI")),
83 AstropyTable(array((
84 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
85 self.testRepo.butler.datastore.root.join(
86 "ingest/run/test_metric_comp.summary/"
87 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
88 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
89 self.testRepo.butler.datastore.root.join(
90 "ingest/run/test_metric_comp.summary/"
91 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))),
92 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
93 "visit", "URI")),
94 )
96 self.assertAstropyTablesEqual(tables, expectedTables)
98 def testNoShowURI(self):
99 """Test for expected output without show_uri (default is False)."""
100 tables = self._queryDatasets(repo=self.root)
102 expectedTables = (
103 AstropyTable(array((
104 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
105 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
106 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
107 ),
108 )
110 self.assertAstropyTablesEqual(tables, expectedTables)
112 def testWhere(self):
113 """Test using the where clause to reduce the number of rows returned.
114 """
115 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423")
117 expectedTables = (
118 AstropyTable(array(
119 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")),
120 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"),
121 ),
122 )
124 self.assertAstropyTablesEqual(tables, expectedTables)
126 def testGlobDatasetType(self):
127 """Test specifying dataset type."""
128 # Create and register an additional DatasetType
130 self.testRepo.butler.registry.insertDimensionData("visit",
131 {"instrument": "DummyCamComp", "id": 425,
132 "name": "fourtwentyfive", "physical_filter": "d-r",
133 "visit_system": 1})
135 datasetType = addDatasetType(self.testRepo.butler,
136 "alt_test_metric_comp",
137 ("instrument", "visit"),
138 "StructuredCompositeReadComp")
140 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
142 # verify the new dataset type increases the number of tables found:
143 tables = self._queryDatasets(repo=self.root)
145 expectedTables = (
146 AstropyTable(array((
147 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
148 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
149 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
150 ),
151 AstropyTable(array((
152 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))),
153 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
154 )
155 )
157 self.assertAstropyTablesEqual(tables, expectedTables)
159 def testFindFirstAndCollections(self):
160 """Test the find-first option, and the collections option, since it
161 is required for find-first."""
163 # Add a new run, and add a dataset to shadow an existing dataset.
164 self.testRepo.addDataset(run="foo",
165 dataId={"instrument": "DummyCamComp", "visit": 424})
167 # Verify that without find-first, duplicate datasets are returned
168 tables = self._queryDatasets(repo=self.root,
169 collections=["foo", "ingest/run"],
170 show_uri=True)
172 expectedTables = (
173 AstropyTable(array(
174 (
175 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
176 self.testRepo.butler.datastore.root.join(
177 "foo/test_metric_comp.data/"
178 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
179 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
180 self.testRepo.butler.datastore.root.join(
181 "ingest/run/test_metric_comp.data/"
182 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
183 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
184 self.testRepo.butler.datastore.root.join(
185 "ingest/run/test_metric_comp.data/"
186 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
187 )),
188 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
189 "visit", "URI")),
190 AstropyTable(array(
191 (
192 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
193 self.testRepo.butler.datastore.root.join(
194 "foo/test_metric_comp.output/"
195 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
196 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
197 self.testRepo.butler.datastore.root.join(
198 "ingest/run/test_metric_comp.output/"
199 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
200 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
201 self.testRepo.butler.datastore.root.join(
202 "ingest/run/test_metric_comp.output/"
203 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
204 )),
205 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
206 "visit", "URI")),
207 AstropyTable(array(
208 (
209 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
210 self.testRepo.butler.datastore.root.join(
211 "foo/test_metric_comp.summary/"
212 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
213 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
214 self.testRepo.butler.datastore.root.join(
215 "ingest/run/test_metric_comp.summary/"
216 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
217 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
218 self.testRepo.butler.datastore.root.join(
219 "ingest/run/test_metric_comp.summary/"
220 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
221 )),
222 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
223 "visit", "URI")),
224 )
226 self.assertAstropyTablesEqual(tables, expectedTables)
228 # Verify that with find first the duplicate dataset is eliminated and
229 # the more recent dataset is returned.
230 tables = self._queryDatasets(repo=self.root,
231 collections=["foo", "ingest/run"],
232 show_uri=True,
233 find_first=True)
235 expectedTables = (
236 AstropyTable(array(
237 (
238 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
239 self.testRepo.butler.datastore.root.join(
240 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
241 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
242 self.testRepo.butler.datastore.root.join(
243 "ingest/run/test_metric_comp.data/"
244 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
245 )),
246 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
247 "visit", "URI")),
248 AstropyTable(array(
249 (
250 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
251 self.testRepo.butler.datastore.root.join(
252 "foo/test_metric_comp.output/"
253 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
254 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
255 self.testRepo.butler.datastore.root.join(
256 "ingest/run/test_metric_comp.output/"
257 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
258 )),
259 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
260 "visit", "URI")),
261 AstropyTable(array(
262 (
263 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
264 self.testRepo.butler.datastore.root.join(
265 "foo/test_metric_comp.summary/"
266 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
267 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
268 self.testRepo.butler.datastore.root.join(
269 "ingest/run/test_metric_comp.summary/"
270 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
271 )),
272 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
273 "visit", "URI")),
274 )
276 self.assertAstropyTablesEqual(tables, expectedTables)
279if __name__ == "__main__": 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true
280 unittest.main()