Coverage for tests/test_cliCmdQueryDatasets.py : 28%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-collections command.
23"""
25import astropy
26from astropy.table import Table as AstropyTable
27from astropy.utils.diff import report_diff_values
28from numpy import array
29import io
30import os
31import shutil
32import tempfile
33import unittest
35from lsst.daf.butler import (
36 Butler,
37 Config,
38 DatasetRef,
39 DatasetType,
40 StorageClassFactory
41)
42from lsst.daf.butler import script
43from lsst.daf.butler.tests import MetricsExample
46TESTDIR = os.path.abspath(os.path.dirname(__file__))
49class QueryDatasetsTest(unittest.TestCase):
51 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
52 storageClassFactory = StorageClassFactory()
54 def _assertTablesEqual(self, tables, expectedTables):
55 """Verify that a list of astropy tables matches a list of expected
56 astropy tables."""
57 diff = io.StringIO()
58 self.assertEqual(len(tables), len(expectedTables))
59 for table, expected in zip(tables, expectedTables):
60 # Assert that we are testing what we think we are testing
61 self.assertIsInstance(table, AstropyTable)
62 self.assertIsInstance(expected, AstropyTable)
63 self.assertTrue(report_diff_values(table, expected, fileobj=diff), msg=diff.getvalue())
65 @staticmethod
66 def _makeExampleMetrics():
67 return MetricsExample({"AM1": 5.2, "AM2": 30.6},
68 {"a": [1, 2, 3],
69 "b": {"blue": 5, "red": "green"}},
70 [563, 234, 456.7, 752, 8, 9, 27])
72 @staticmethod
73 def _addDatasetType(datasetTypeName, dimensions, storageClass, registry):
74 """Create a DatasetType and register it
75 """
76 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
77 registry.registerDatasetType(datasetType)
78 return datasetType
80 @staticmethod
81 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
82 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri)
84 def setUp(self):
85 self.root = tempfile.mkdtemp(dir=TESTDIR)
86 Butler.makeRepo(self.root, config=Config(self.configFile))
87 self.butlerConfigFile = os.path.join(self.root, "butler.yaml")
88 self.storageClassFactory.addFromConfig(self.configFile)
90 # New datasets will be added to run and tag, but we will only look in
91 # tag when looking up datasets.
92 run = "ingest/run"
93 tag = "ingest"
94 self.butler = Butler(self.butlerConfigFile, run=run, collections=[tag], tags=[tag])
96 # There will not be a collection yet
97 collections = set(self.butler.registry.queryCollections())
98 self.assertEqual(collections, set([run, tag]))
100 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp")
102 # Create and register a DatasetType
103 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"])
104 datasetTypeName = "test_metric_comp"
105 self.datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass,
106 self.butler.registry)
108 # Add needed Dimensions
109 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"})
110 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp",
111 "name": "d-r",
112 "band": "R"})
113 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp",
114 "id": 1,
115 "name": "default"})
116 visit_start = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai")
117 visit_end = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai")
118 self.butler.registry.insertDimensionData("visit",
119 {"instrument": "DummyCamComp", "id": 423,
120 "name": "fourtwentythree", "physical_filter": "d-r",
121 "visit_system": 1, "datetime_begin": visit_start,
122 "datetime_end": visit_end})
123 self.butler.registry.insertDimensionData("visit", {"instrument": "DummyCamComp", "id": 424,
124 "name": "fourtwentyfour", "physical_filter": "d-r",
125 "visit_system": 1})
126 metric = self._makeExampleMetrics()
127 dataId = {"instrument": "DummyCamComp", "visit": 423}
128 ref = DatasetRef(self.datasetType, dataId, id=None)
129 self.butler.put(metric, ref)
131 metric = self._makeExampleMetrics()
132 dataId = {"instrument": "DummyCamComp", "visit": 424}
133 ref = DatasetRef(self.datasetType, dataId, id=None)
134 self.butler.put(metric, ref)
136 def tearDown(self):
137 if os.path.exists(self.root):
138 shutil.rmtree(self.root, ignore_errors=True)
140 def testShowURI(self):
141 """Test for expected output with show_uri=True."""
142 tables = self._queryDatasets(repo=self.butlerConfigFile, show_uri=True)
144 expectedTables = (
145 AstropyTable(array((
146 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
147 self.butler.datastore.root.join(
148 "ingest/run/test_metric_comp.data/"
149 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
150 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
151 self.butler.datastore.root.join(
152 "ingest/run/test_metric_comp.data/"
153 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))),
154 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
155 "visit", "URI")),
156 AstropyTable(array((
157 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
158 self.butler.datastore.root.join(
159 "ingest/run/test_metric_comp.output/"
160 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
161 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
162 self.butler.datastore.root.join(
163 "ingest/run/test_metric_comp.output/"
164 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))),
165 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
166 "visit", "URI")),
167 AstropyTable(array((
168 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
169 self.butler.datastore.root.join(
170 "ingest/run/test_metric_comp.summary/"
171 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
172 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
173 self.butler.datastore.root.join(
174 "ingest/run/test_metric_comp.summary/"
175 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))),
176 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
177 "visit", "URI")),
178 )
180 self._assertTablesEqual(tables, expectedTables)
182 def testNoShowURI(self):
183 """Test for expected output without show_uri (default is False)."""
184 tables = self._queryDatasets(repo=self.butlerConfigFile)
186 expectedTables = (
187 AstropyTable(array((
188 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
189 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
190 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
191 ),
192 )
194 self._assertTablesEqual(tables, expectedTables)
196 def testWhere(self):
197 """Test using the where clause to reduce the number of rows returned.
198 """
199 tables = self._queryDatasets(repo=self.butlerConfigFile, where="visit=423")
201 expectedTables = (
202 AstropyTable(array(
203 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")),
204 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"),
205 ),
206 )
208 self._assertTablesEqual(tables, expectedTables)
210 def testGlobDatasetType(self):
211 """Test specifying dataset type."""
212 # Create and register an additional DatasetType
213 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"])
214 datasetTypeName = "alt_test_metric_comp"
215 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp")
216 datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass, self.butler.registry)
217 self.butler.registry.insertDimensionData(
218 "visit", {"instrument": "DummyCamComp", "id": 425,
219 "name": "fourtwentyfive", "physical_filter": "d-r",
220 "visit_system": 1})
221 metric = self._makeExampleMetrics()
222 dataId = {"instrument": "DummyCamComp", "visit": 425}
223 ref = DatasetRef(datasetType, dataId, id=None)
224 self.butler.put(metric, ref)
226 # verify the new dataset type increases the number of tables found:
227 tables = self._queryDatasets(repo=self.butlerConfigFile)
229 expectedTables = (
230 AstropyTable(array((
231 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
232 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
233 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
234 ),
235 AstropyTable(array((
236 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))),
237 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
238 )
239 )
241 self._assertTablesEqual(tables, expectedTables)
243 def testFindFirstAndCollections(self):
244 """Test the find-first option, and the collections option, since it
245 is required for find-first."""
247 # Create a new butler with a new run, and add a dataset to shadow an
248 # existing dataset.
249 self.butler = Butler(self.root, run="foo")
250 metric = self._makeExampleMetrics()
251 dataId = {"instrument": "DummyCamComp", "visit": 424}
252 ref = DatasetRef(self.datasetType, dataId, id=None)
253 self.butler.put(metric, ref)
255 # Verify that without find-first, duplicate datasets are returned
256 tables = self._queryDatasets(repo=self.butlerConfigFile,
257 collections=["foo", "ingest/run"],
258 show_uri=True)
260 expectedTables = (
261 AstropyTable(array(
262 (
263 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
264 self.butler.datastore.root.join(
265 "foo/test_metric_comp.data/"
266 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
267 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
268 self.butler.datastore.root.join(
269 "ingest/run/test_metric_comp.data/"
270 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
271 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
272 self.butler.datastore.root.join(
273 "ingest/run/test_metric_comp.data/"
274 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
275 )),
276 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
277 "visit", "URI")),
278 AstropyTable(array(
279 (
280 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
281 self.butler.datastore.root.join(
282 "foo/test_metric_comp.output/"
283 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
284 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
285 self.butler.datastore.root.join(
286 "ingest/run/test_metric_comp.output/"
287 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
288 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
289 self.butler.datastore.root.join(
290 "ingest/run/test_metric_comp.output/"
291 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
292 )),
293 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
294 "visit", "URI")),
295 AstropyTable(array(
296 (
297 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
298 self.butler.datastore.root.join(
299 "foo/test_metric_comp.summary/"
300 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
301 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
302 self.butler.datastore.root.join(
303 "ingest/run/test_metric_comp.summary/"
304 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
305 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
306 self.butler.datastore.root.join(
307 "ingest/run/test_metric_comp.summary/"
308 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
309 )),
310 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
311 "visit", "URI")),
312 )
314 self._assertTablesEqual(tables, expectedTables)
316 # Verify that with find first the duplicate dataset is eliminated and
317 # the more recent dataset is returned.
318 tables = self._queryDatasets(repo=self.butlerConfigFile,
319 collections=["foo", "ingest/run"],
320 show_uri=True,
321 find_first=True)
323 expectedTables = (
324 AstropyTable(array(
325 (
326 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
327 self.butler.datastore.root.join(
328 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
329 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
330 self.butler.datastore.root.join(
331 "ingest/run/test_metric_comp.data/"
332 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
333 )),
334 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
335 "visit", "URI")),
336 AstropyTable(array(
337 (
338 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
339 self.butler.datastore.root.join(
340 "foo/test_metric_comp.output/"
341 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
342 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
343 self.butler.datastore.root.join(
344 "ingest/run/test_metric_comp.output/"
345 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
346 )),
347 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
348 "visit", "URI")),
349 AstropyTable(array(
350 (
351 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
352 self.butler.datastore.root.join(
353 "foo/test_metric_comp.summary/"
354 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
355 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
356 self.butler.datastore.root.join(
357 "ingest/run/test_metric_comp.summary/"
358 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
359 )),
360 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
361 "visit", "URI")),
362 )
364 self._assertTablesEqual(tables, expectedTables)
367if __name__ == "__main__": 367 ↛ 368line 367 didn't jump to line 368, because the condition on line 367 was never true
368 unittest.main()