Coverage for tests/test_cliCmdQueryDatasets.py : 29%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI query-collections command.
23"""
25import astropy
26from astropy.table import Table as AstropyTable
27from numpy import array
28import os
29import shutil
30import tempfile
31import unittest
33from lsst.daf.butler import (
34 Butler,
35 Config,
36 DatasetRef,
37 DatasetType,
38 StorageClassFactory
39)
40from lsst.daf.butler import script
41from lsst.daf.butler.tests import MetricsExample
42from lsst.daf.butler.tests.utils import ButlerTestHelper
45TESTDIR = os.path.abspath(os.path.dirname(__file__))
48class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
50 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
51 storageClassFactory = StorageClassFactory()
53 @staticmethod
54 def _makeExampleMetrics():
55 return MetricsExample({"AM1": 5.2, "AM2": 30.6},
56 {"a": [1, 2, 3],
57 "b": {"blue": 5, "red": "green"}},
58 [563, 234, 456.7, 752, 8, 9, 27])
60 @staticmethod
61 def _addDatasetType(datasetTypeName, dimensions, storageClass, registry):
62 """Create a DatasetType and register it
63 """
64 datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
65 registry.registerDatasetType(datasetType)
66 return datasetType
68 @staticmethod
69 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
70 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri)
72 def setUp(self):
73 self.root = tempfile.mkdtemp(dir=TESTDIR)
74 Butler.makeRepo(self.root, config=Config(self.configFile))
75 self.butlerConfigFile = os.path.join(self.root, "butler.yaml")
76 self.storageClassFactory.addFromConfig(self.configFile)
78 # New datasets will be added to run and tag, but we will only look in
79 # tag when looking up datasets.
80 run = "ingest/run"
81 tag = "ingest"
82 self.butler = Butler(self.butlerConfigFile, run=run, collections=[tag], tags=[tag])
84 # There will not be a collection yet
85 collections = set(self.butler.registry.queryCollections())
86 self.assertEqual(collections, set([run, tag]))
88 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp")
90 # Create and register a DatasetType
91 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"])
92 datasetTypeName = "test_metric_comp"
93 self.datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass,
94 self.butler.registry)
96 # Add needed Dimensions
97 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"})
98 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp",
99 "name": "d-r",
100 "band": "R"})
101 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp",
102 "id": 1,
103 "name": "default"})
104 visit_start = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai")
105 visit_end = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai")
106 self.butler.registry.insertDimensionData("visit",
107 {"instrument": "DummyCamComp", "id": 423,
108 "name": "fourtwentythree", "physical_filter": "d-r",
109 "visit_system": 1, "datetime_begin": visit_start,
110 "datetime_end": visit_end})
111 self.butler.registry.insertDimensionData("visit", {"instrument": "DummyCamComp", "id": 424,
112 "name": "fourtwentyfour", "physical_filter": "d-r",
113 "visit_system": 1})
114 metric = self._makeExampleMetrics()
115 dataId = {"instrument": "DummyCamComp", "visit": 423}
116 ref = DatasetRef(self.datasetType, dataId, id=None)
117 self.butler.put(metric, ref)
119 metric = self._makeExampleMetrics()
120 dataId = {"instrument": "DummyCamComp", "visit": 424}
121 ref = DatasetRef(self.datasetType, dataId, id=None)
122 self.butler.put(metric, ref)
124 def tearDown(self):
125 if os.path.exists(self.root):
126 shutil.rmtree(self.root, ignore_errors=True)
128 def testShowURI(self):
129 """Test for expected output with show_uri=True."""
130 tables = self._queryDatasets(repo=self.butlerConfigFile, show_uri=True)
132 expectedTables = (
133 AstropyTable(array((
134 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
135 self.butler.datastore.root.join(
136 "ingest/run/test_metric_comp.data/"
137 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
138 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
139 self.butler.datastore.root.join(
140 "ingest/run/test_metric_comp.data/"
141 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))),
142 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
143 "visit", "URI")),
144 AstropyTable(array((
145 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
146 self.butler.datastore.root.join(
147 "ingest/run/test_metric_comp.output/"
148 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
149 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
150 self.butler.datastore.root.join(
151 "ingest/run/test_metric_comp.output/"
152 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))),
153 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
154 "visit", "URI")),
155 AstropyTable(array((
156 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
157 self.butler.datastore.root.join(
158 "ingest/run/test_metric_comp.summary/"
159 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
160 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
161 self.butler.datastore.root.join(
162 "ingest/run/test_metric_comp.summary/"
163 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))),
164 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
165 "visit", "URI")),
166 )
168 self.assertAstropyTablesEqual(tables, expectedTables)
170 def testNoShowURI(self):
171 """Test for expected output without show_uri (default is False)."""
172 tables = self._queryDatasets(repo=self.butlerConfigFile)
174 expectedTables = (
175 AstropyTable(array((
176 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
177 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
178 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
179 ),
180 )
182 self.assertAstropyTablesEqual(tables, expectedTables)
184 def testWhere(self):
185 """Test using the where clause to reduce the number of rows returned.
186 """
187 tables = self._queryDatasets(repo=self.butlerConfigFile, where="visit=423")
189 expectedTables = (
190 AstropyTable(array(
191 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")),
192 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"),
193 ),
194 )
196 self.assertAstropyTablesEqual(tables, expectedTables)
198 def testGlobDatasetType(self):
199 """Test specifying dataset type."""
200 # Create and register an additional DatasetType
201 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"])
202 datasetTypeName = "alt_test_metric_comp"
203 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp")
204 datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass, self.butler.registry)
205 self.butler.registry.insertDimensionData(
206 "visit", {"instrument": "DummyCamComp", "id": 425,
207 "name": "fourtwentyfive", "physical_filter": "d-r",
208 "visit_system": 1})
209 metric = self._makeExampleMetrics()
210 dataId = {"instrument": "DummyCamComp", "visit": 425}
211 ref = DatasetRef(datasetType, dataId, id=None)
212 self.butler.put(metric, ref)
214 # verify the new dataset type increases the number of tables found:
215 tables = self._queryDatasets(repo=self.butlerConfigFile)
217 expectedTables = (
218 AstropyTable(array((
219 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
220 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
221 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
222 ),
223 AstropyTable(array((
224 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))),
225 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
226 )
227 )
229 self.assertAstropyTablesEqual(tables, expectedTables)
231 def testFindFirstAndCollections(self):
232 """Test the find-first option, and the collections option, since it
233 is required for find-first."""
235 # Create a new butler with a new run, and add a dataset to shadow an
236 # existing dataset.
237 self.butler = Butler(self.root, run="foo")
238 metric = self._makeExampleMetrics()
239 dataId = {"instrument": "DummyCamComp", "visit": 424}
240 ref = DatasetRef(self.datasetType, dataId, id=None)
241 self.butler.put(metric, ref)
243 # Verify that without find-first, duplicate datasets are returned
244 tables = self._queryDatasets(repo=self.butlerConfigFile,
245 collections=["foo", "ingest/run"],
246 show_uri=True)
248 expectedTables = (
249 AstropyTable(array(
250 (
251 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
252 self.butler.datastore.root.join(
253 "foo/test_metric_comp.data/"
254 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
255 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
256 self.butler.datastore.root.join(
257 "ingest/run/test_metric_comp.data/"
258 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
259 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
260 self.butler.datastore.root.join(
261 "ingest/run/test_metric_comp.data/"
262 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
263 )),
264 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
265 "visit", "URI")),
266 AstropyTable(array(
267 (
268 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
269 self.butler.datastore.root.join(
270 "foo/test_metric_comp.output/"
271 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
272 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
273 self.butler.datastore.root.join(
274 "ingest/run/test_metric_comp.output/"
275 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
276 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
277 self.butler.datastore.root.join(
278 "ingest/run/test_metric_comp.output/"
279 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
280 )),
281 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
282 "visit", "URI")),
283 AstropyTable(array(
284 (
285 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
286 self.butler.datastore.root.join(
287 "foo/test_metric_comp.summary/"
288 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
289 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
290 self.butler.datastore.root.join(
291 "ingest/run/test_metric_comp.summary/"
292 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
293 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
294 self.butler.datastore.root.join(
295 "ingest/run/test_metric_comp.summary/"
296 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
297 )),
298 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
299 "visit", "URI")),
300 )
302 self.assertAstropyTablesEqual(tables, expectedTables)
304 # Verify that with find first the duplicate dataset is eliminated and
305 # the more recent dataset is returned.
306 tables = self._queryDatasets(repo=self.butlerConfigFile,
307 collections=["foo", "ingest/run"],
308 show_uri=True,
309 find_first=True)
311 expectedTables = (
312 AstropyTable(array(
313 (
314 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
315 self.butler.datastore.root.join(
316 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
317 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
318 self.butler.datastore.root.join(
319 "ingest/run/test_metric_comp.data/"
320 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
321 )),
322 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
323 "visit", "URI")),
324 AstropyTable(array(
325 (
326 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
327 self.butler.datastore.root.join(
328 "foo/test_metric_comp.output/"
329 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
330 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
331 self.butler.datastore.root.join(
332 "ingest/run/test_metric_comp.output/"
333 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
334 )),
335 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
336 "visit", "URI")),
337 AstropyTable(array(
338 (
339 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
340 self.butler.datastore.root.join(
341 "foo/test_metric_comp.summary/"
342 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
343 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
344 self.butler.datastore.root.join(
345 "ingest/run/test_metric_comp.summary/"
346 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
347 )),
348 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
349 "visit", "URI")),
350 )
352 self.assertAstropyTablesEqual(tables, expectedTables)
355if __name__ == "__main__": 355 ↛ 356line 355 didn't jump to line 356, because the condition on line 355 was never true
356 unittest.main()