Coverage for tests/test_cliCmdPruneDatasets.py : 40%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Unit tests for daf_butler CLI prune-datasets subcommand.
23"""
25from astropy.table import Table
26import unittest
27from unittest.mock import patch
29from lsst.daf.butler import Butler
30from lsst.daf.butler.cli.butler import cli as butlerCli
31from lsst.daf.butler.cli.cmd.commands import (
32 pruneDatasets_wouldRemoveMsg,
33 pruneDatasets_wouldDisassociateMsg,
34 pruneDatasets_wouldDisassociateAndRemoveMsg,
35 pruneDatasets_willRemoveMsg,
36 pruneDatasets_askContinueMsg,
37 pruneDatasets_didRemoveAforementioned,
38 pruneDatasets_didNotRemoveAforementioned,
39 pruneDatasets_didRemoveMsg,
40 pruneDatasets_noDatasetsFound,
41 pruneDatasets_errPurgeAndDisassociate,
42 pruneDatasets_errQuietWithDryRun,
43 pruneDatasets_errNoCollectionRestriction,
44 pruneDatasets_errPruneOnNotRun,
45)
46from lsst.daf.butler.cli.utils import astropyTablesToStr, clickResultMsg, LogCliRunner
47from lsst.daf.butler.registry import CollectionType
48import lsst.daf.butler.script
49from lsst.daf.butler.script import QueryDatasets
51# Tests require the SqlRegistry
52import lsst.daf.butler.registries.sql
54doFindTables = True
57def getTables():
58 if doFindTables:
59 return (Table(((1, 2, 3),), names=("foo",)),)
60 return tuple()
63def getDatasets():
64 return "datasets"
67def makeQueryDatasets(*args, **kwargs):
68 return QueryDatasets(*args, **kwargs)
71class PruneDatasetsTestCase(unittest.TestCase):
72 """Tests the ``prune_datasets`` "command" function (in
73 ``cli/cmd/commands.py``) and the ``pruneDatasets`` "script" function (in
74 ``scripts/_pruneDatasets.py``).
76 ``Butler.pruneDatasets`` and a few other functions that get called before
77 it are mocked, and tests check for expected arguments to those mocks.
78 """
80 def setUp(self):
81 self.repo = "here"
83 @staticmethod
84 def makeQueryDatasetsArgs(*, repo, **kwargs):
85 expectedArgs = dict(repo=repo, collections=(), where=None, find_first=True, show_uri=False,
86 glob=tuple())
87 expectedArgs.update(kwargs)
88 return expectedArgs
90 @staticmethod
91 def makePruneDatasetsArgs(**kwargs):
92 expectedArgs = dict(refs=tuple(), disassociate=False, tags=(), purge=False, run=None,
93 unstore=False)
94 expectedArgs.update(kwargs)
95 return expectedArgs
97 # Mock the QueryDatasets.getTables function to return a set of Astropy
98 # tables, similar to what would be returned by a call to
99 # QueryDatasets.getTables on a repo with real data.
100 @patch.object(lsst.daf.butler.script._pruneDatasets.QueryDatasets, "getTables", side_effect=getTables)
101 # Mock the QueryDatasets.getDatasets function. Normally it would return a
102 # list of queries.DatasetQueryResults, but all we need to do is verify that
103 # the output of this function is passed into our pruneDatasets magicMock,
104 # so we can return something arbitrary that we can test is equal."""
105 @patch.object(lsst.daf.butler.script._pruneDatasets.QueryDatasets, "getDatasets", side_effect=getDatasets)
106 # Mock the actual QueryDatasets class, so we can inspect calls to its init
107 # function. Note that the side_effect returns an instance of QueryDatasets,
108 # so this mock records and then is a pass-through.
109 @patch.object(lsst.daf.butler.script._pruneDatasets, "QueryDatasets", side_effect=makeQueryDatasets)
110 # Mock the pruneDatasets butler command so we can test for expected calls
111 # to it, without dealing with setting up a full repo with data for it.
112 @patch.object(Butler, "pruneDatasets")
113 def run_test(self,
114 mockPruneDatasets,
115 mockQueryDatasets_init,
116 mockQueryDatasets_getDatasets,
117 mockQueryDatasets_getTables,
118 cliArgs,
119 exMsgs,
120 exPruneDatasetsCallArgs,
121 exGetTablesCalled,
122 exQueryDatasetsCallArgs,
123 invokeInput=None,
124 exPruneDatasetsExitCode=0):
125 """Execute the test.
127 Makes a temporary repo, invokes ``prune-datasets``. Verifies expected
128 output, exit codes, and mock calls.
130 Parameters
131 ----------
132 mockPruneDatasets : `MagicMock`
133 The MagicMock for the ``Butler.pruneDatasets`` function.
134 mockQueryDatasets_init : `MagicMock`
135 The MagicMock for the ``QueryDatasets.__init__`` function.
136 mockQueryDatasets_getDatasets : `MagicMock`
137 The MagicMock for the ``QueryDatasets.getDatasets`` function.
138 mockQueryDatasets_getTables : `MagicMock`
139 The MagicMock for the ``QueryDatasets.getTables`` function.
140 cliArgs : `list` [`str`]
141 The arguments to pass to the command line. Do not include the
142 subcommand name or the repo.
143 exMsgs : `list` [`str`] or None
144 A list of text fragments that should appear in the text output
145 after calling the CLI command, or None if no output should be
146 produced.
147 exPruneDatasetsCallArgs : `dict` [`str`, `Any`]
148 The arguments that ``Butler.pruneDatasets`` should have been called
149 with, or None if that function should not have been called.
150 exGetTablesCalled : bool
151 `True` if ``QueryDatasets.getTables`` should have been called, else
152 `False`.
153 exQueryDatasetsCallArgs : `dict` [`str`, `Any`]
154 The arguments that ``QueryDatasets.__init__`` should have bene
155 called with, or `None` if the function should not have been called.
156 invokeInput : `str`, optional.
157 As string to pass to the ``CliRunner.invoke`` `input` argument. By
158 default None.
159 exPruneDatasetsExitCode : `int`
160 The expected exit code returned from invoking ``prune-datasets``.
161 """
162 runner = LogCliRunner()
163 with runner.isolated_filesystem():
164 # Make a repo so a butler can be created
165 result = runner.invoke(butlerCli, ["create", self.repo])
166 self.assertEqual(result.exit_code, 0, clickResultMsg(result))
168 # Run the prune-datasets CLI command, this will call all of our
169 # mocks:
170 cliArgs = ["prune-datasets", self.repo] + cliArgs
171 result = runner.invoke(butlerCli, cliArgs, input=invokeInput)
172 self.assertEqual(result.exit_code, exPruneDatasetsExitCode, clickResultMsg(result))
174 # Verify the Butler.pruneDatasets was called exactly once with
175 # expected arguments. The datasets argument is the value returned
176 # by QueryDatasets, which we've mocked with side effect
177 # ``getDatasets()``.
178 if exPruneDatasetsCallArgs:
179 mockPruneDatasets.assert_called_once_with(**exPruneDatasetsCallArgs)
180 else:
181 mockPruneDatasets.assert_not_called()
183 # Less critical, but do a quick verification that the QueryDataset
184 # member function mocks were called, in this case we expect one
185 # time each.
186 if exQueryDatasetsCallArgs:
187 mockQueryDatasets_init.assert_called_once_with(**exQueryDatasetsCallArgs)
188 else:
189 mockQueryDatasets_init.assert_not_called()
190 # If Butler.pruneDatasets was not called, then
191 # QueryDatasets.getDatasets also does not get called.
192 if exPruneDatasetsCallArgs:
193 mockQueryDatasets_getDatasets.assert_called_once()
194 else:
195 mockQueryDatasets_getDatasets.assert_not_called()
196 if exGetTablesCalled:
197 mockQueryDatasets_getTables.assert_called_once()
198 else:
199 mockQueryDatasets_getTables.assert_not_called()
201 if exMsgs is None:
202 self.assertEqual("", result.output)
203 else:
204 for expectedMsg in exMsgs:
205 self.assertIn(expectedMsg, result.output)
207 def test_defaults_doContinue(self):
208 """Test running with the default values.
210 Verify that with the default flags that the subcommand says what it
211 will do, prompts for input, and says that it's done."""
212 self.run_test(cliArgs=["myCollection", "--unstore"],
213 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(refs=getDatasets(),
214 unstore=True),
215 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
216 collections=("myCollection",)),
217 exGetTablesCalled=True,
218 exMsgs=(pruneDatasets_willRemoveMsg,
219 pruneDatasets_askContinueMsg,
220 astropyTablesToStr(getTables()),
221 pruneDatasets_didRemoveAforementioned),
222 invokeInput="yes")
224 def test_defaults_doNotContinue(self):
225 """Test running with the default values but not continuing.
227 Verify that with the default flags that the subcommand says what it
228 will do, prompts for input, and aborts when told not to continue."""
229 self.run_test(cliArgs=["myCollection", "--unstore"],
230 exPruneDatasetsCallArgs=None,
231 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
232 collections=("myCollection",)),
233 exGetTablesCalled=True,
234 exMsgs=(pruneDatasets_willRemoveMsg,
235 pruneDatasets_askContinueMsg,
236 pruneDatasets_didNotRemoveAforementioned),
237 invokeInput="no")
239 def test_dryRun_unstore(self):
240 """Test the --dry-run flag with --unstore.
242 Verify that with the dry-run flag the subcommand says what it would
243 remove, but does not remove the datasets."""
244 self.run_test(cliArgs=["myCollection", "--dry-run", "--unstore"],
245 exPruneDatasetsCallArgs=None,
246 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
247 collections=("myCollection",)),
248 exGetTablesCalled=True,
249 exMsgs=(pruneDatasets_wouldRemoveMsg,
250 astropyTablesToStr(getTables())))
252 def test_dryRun_disassociate(self):
253 """Test the --dry-run flag with --disassociate.
255 Verify that with the dry-run flag the subcommand says what it would
256 remove, but does not remove the datasets. """
257 collection = "myCollection"
258 self.run_test(cliArgs=[collection, "--dry-run", "--disassociate", "tag1"],
259 exPruneDatasetsCallArgs=None,
260 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
261 collections=(collection,)),
262 exGetTablesCalled=True,
263 exMsgs=(pruneDatasets_wouldDisassociateMsg.format(collections=(collection,)),
264 astropyTablesToStr(getTables())))
266 def test_dryRun_unstoreAndDisassociate(self):
267 """Test the --dry-run flag with --unstore and --disassociate.
269 Verify that with the dry-run flag the subcommand says what it would
270 remove, but does not remove the datasets. """
271 collection = "myCollection"
272 self.run_test(cliArgs=[collection, "--dry-run", "--unstore", "--disassociate", "tag1"],
273 exPruneDatasetsCallArgs=None,
274 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
275 collections=(collection,)),
276 exGetTablesCalled=True,
277 exMsgs=(pruneDatasets_wouldDisassociateAndRemoveMsg.format(collections=(collection,)),
278 astropyTablesToStr(getTables())))
280 def test_noConfirm(self):
281 """Test the --no-confirm flag.
283 Verify that with the no-confirm flag the subcommand does not ask for
284 a confirmation, prints the did remove message and the tables that were
285 passed for removal."""
286 self.run_test(cliArgs=["myCollection", "--no-confirm", "--unstore"],
287 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(refs=getDatasets(),
288 unstore=True),
289 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
290 collections=("myCollection",)),
291 exGetTablesCalled=True,
292 exMsgs=(pruneDatasets_didRemoveMsg,
293 astropyTablesToStr(getTables())))
295 def test_quiet(self):
296 """Test the --quiet flag.
298 Verify that with the quiet flag and the no-confirm flags set that no
299 output is produced by the subcommand."""
300 self.run_test(cliArgs=["myCollection", "--quiet", "--unstore"],
301 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(refs=getDatasets(), unstore=True),
302 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
303 collections=("myCollection",)),
304 exGetTablesCalled=True,
305 exMsgs=None)
307 def test_quietWithDryRun(self):
308 """Test for an error using the --quiet flag with --dry-run.
309 """
310 self.run_test(cliArgs=["--quiet", "--dry-run", "--unstore"],
311 exPruneDatasetsCallArgs=None,
312 exQueryDatasetsCallArgs=None,
313 exGetTablesCalled=False,
314 exMsgs=(pruneDatasets_errQuietWithDryRun,),
315 exPruneDatasetsExitCode=1)
317 def test_noCollections(self):
318 """Test for an error if no collections are indicated.
319 """
320 self.run_test(cliArgs=["--find-all", "--unstore"],
321 exPruneDatasetsCallArgs=None,
322 exQueryDatasetsCallArgs=None,
323 exGetTablesCalled=False,
324 exMsgs=(pruneDatasets_errNoCollectionRestriction,),
325 exPruneDatasetsExitCode=1)
327 def test_noDatasets(self):
328 """Test for expected outputs when no datasets are found."""
329 global doFindTables
330 reset = doFindTables
331 try:
332 doFindTables = False
333 self.run_test(cliArgs=["myCollection", "--unstore"],
334 exPruneDatasetsCallArgs=None,
335 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
336 collections=("myCollection",)),
337 exGetTablesCalled=True,
338 exMsgs=(pruneDatasets_noDatasetsFound,))
339 finally:
340 doFindTables = reset
342 def test_purgeWithDisassociate(self):
343 """Verify there is an error when --purge and --disassociate are both
344 passed in. """
345 self.run_test(
346 cliArgs=["--purge", "run", "--disassociate", "tag1", "tag2"],
347 exPruneDatasetsCallArgs=None,
348 exQueryDatasetsCallArgs=None, # should not make it far enough to call this.
349 exGetTablesCalled=False, # ...or this.
350 exMsgs=(pruneDatasets_errPurgeAndDisassociate,),
351 exPruneDatasetsExitCode=1
352 )
354 @patch.object(lsst.daf.butler.registries.sql.SqlRegistry, "getCollectionType", 354 ↛ exitline 354 didn't jump to the function exit
355 side_effect=lambda x: CollectionType.RUN)
356 def test_purgeImpliedArgs(self, mockGetCollectionType):
357 """Verify the arguments implied by --purge.
359 --purge <run> implies the following arguments to butler.pruneDatasets:
360 purge=True, run=<run>, disassociate=True, unstore=True
361 And for QueryDatasets, if COLLECTIONS is not passed then <run> gets
362 used as the value of COLLECTIONS (and when there is a COLLECTIONS
363 value then find_first gets set to True)
364 """
365 self.run_test(
366 cliArgs=["--purge", "run"],
367 invokeInput="yes",
368 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(
369 purge=True,
370 run="run",
371 refs=getDatasets(),
372 disassociate=True,
373 unstore=True
374 ),
375 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
376 collections=("run",),
377 find_first=True),
378 exGetTablesCalled=True,
379 exMsgs=(pruneDatasets_willRemoveMsg,
380 pruneDatasets_askContinueMsg,
381 astropyTablesToStr(getTables()),
382 pruneDatasets_didRemoveAforementioned)
383 )
385 @patch.object(lsst.daf.butler.registries.sql.SqlRegistry, "getCollectionType", 385 ↛ exitline 385 didn't jump to the function exit
386 side_effect=lambda x: CollectionType.RUN)
387 def test_purgeImpliedArgsWithCollections(self, mockGetCollectionType):
388 """Verify the arguments implied by --purge, with a COLLECTIONS."""
389 self.run_test(
390 cliArgs=["myCollection", "--purge", "run"],
391 invokeInput="yes",
392 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(
393 purge=True,
394 run="run",
395 disassociate=True,
396 unstore=True,
397 refs=getDatasets()
398 ),
399 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo, collections=("myCollection",),
400 find_first=True),
401 exGetTablesCalled=True,
402 exMsgs=(pruneDatasets_willRemoveMsg,
403 pruneDatasets_askContinueMsg,
404 astropyTablesToStr(getTables()),
405 pruneDatasets_didRemoveAforementioned)
406 )
408 @patch.object(lsst.daf.butler.registries.sql.SqlRegistry, "getCollectionType", 408 ↛ exitline 408 didn't jump to the function exit
409 side_effect=lambda x: CollectionType.TAGGED)
410 def test_purgeOnNonRunCollection(self, mockGetCollectionType):
411 """Verify calling run on a non-run collection fails with expected
412 error message. """
413 collectionName = "myTaggedCollection"
414 self.run_test(
415 cliArgs=["--purge", collectionName],
416 invokeInput="yes",
417 exPruneDatasetsCallArgs=None,
418 exQueryDatasetsCallArgs=None,
419 exGetTablesCalled=False,
420 exMsgs=(pruneDatasets_errPruneOnNotRun.format(collection=collectionName)),
421 exPruneDatasetsExitCode=1,
422 )
424 def test_disassociateImpliedArgs(self):
425 """Verify the arguments implied by --disassociate.
427 --disassociate <tags> implies the following arguments to
428 butler.pruneDatasets:
429 disassociate=True, tags=<tags>
430 and if COLLECTIONS is not passed then <tags> gets used as the value
431 of COLLECTIONS.
433 Use the --no-confirm flag instead of invokeInput="yes", and check for
434 the associated output.
435 """
436 self.run_test(
437 cliArgs=["--disassociate", "tag1", "--disassociate", "tag2", "--no-confirm"],
438 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(
439 tags=("tag1", "tag2"),
440 disassociate=True,
441 refs=getDatasets()
442 ),
443 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
444 collections=("tag1", "tag2"),
445 find_first=True),
446 exGetTablesCalled=True,
447 exMsgs=(pruneDatasets_didRemoveMsg,
448 astropyTablesToStr(getTables()))
449 )
451 def test_disassociateImpliedArgsWithCollections(self):
452 """Verify the arguments implied by --disassociate, with a --collection
453 flag."""
454 self.run_test(
455 cliArgs=["myCollection", "--disassociate", "tag1", "--disassociate", "tag2", "--no-confirm"],
456 exPruneDatasetsCallArgs=self.makePruneDatasetsArgs(
457 tags=("tag1", "tag2"),
458 disassociate=True,
459 refs=getDatasets()
460 ),
461 exQueryDatasetsCallArgs=self.makeQueryDatasetsArgs(repo=self.repo,
462 collections=("myCollection",),
463 find_first=True),
464 exGetTablesCalled=True,
465 exMsgs=(pruneDatasets_didRemoveMsg,
466 astropyTablesToStr(getTables()))
467 )
470if __name__ == "__main__": 470 ↛ 471line 470 didn't jump to line 471, because the condition on line 470 was never true
471 unittest.main()