Coverage for tests/test_driver.py : 30%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import argparse
25import functools
26import os
27import shutil
28import tempfile
29import unittest.mock
31from lsst.daf.base import PropertySet
32from lsst.pipe.base import DataIdContainer, Struct
33import lsst.utils.tests
34from lsst.ap.pipe import ApPipeTask
35from lsst.ap.verify import pipeline_driver
36from lsst.ap.verify.workspace import WorkspaceGen2, WorkspaceGen3
39def _getDataIds():
40 return [{"visit": 42, "ccd": 0}]
43def patchApPipe(method):
44 """Shortcut decorator for consistently patching ApPipeTask.
45 """
46 @functools.wraps(method)
47 def wrapper(self, *args, **kwargs):
48 parsedCmd = argparse.Namespace()
49 parsedCmd.id = DataIdContainer()
50 parsedCmd.id.idList = _getDataIds()
51 parReturn = Struct(
52 argumentParser=None,
53 parsedCmd=parsedCmd,
54 taskRunner=None,
55 resultList=[Struct(exitStatus=0)])
56 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb")
57 pipePatcher = unittest.mock.patch("lsst.ap.pipe.ApPipeTask",
58 **{"parseAndRun.return_value": parReturn},
59 _DefaultName=ApPipeTask._DefaultName,
60 ConfigClass=ApPipeTask.ConfigClass)
61 patchedMethod = pipePatcher(dbPatcher(method))
62 return patchedMethod(self, *args, **kwargs)
63 return wrapper
66def patchApPipeGen3(method):
67 """Shortcut decorator for consistently patching AP code.
68 """
69 @functools.wraps(method)
70 def wrapper(self, *args, **kwargs):
71 parsedCmd = argparse.Namespace()
72 parsedCmd.id = DataIdContainer()
73 parsedCmd.id.idList = _getDataIds()
74 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb")
75 execPatcher = unittest.mock.patch("lsst.ctrl.mpexec.CmdLineFwk")
76 patchedMethod = execPatcher(dbPatcher(method))
77 return patchedMethod(self, *args, **kwargs)
78 return wrapper
81class PipelineDriverTestSuiteGen2(lsst.utils.tests.TestCase):
82 def setUp(self):
83 self._testDir = tempfile.mkdtemp()
84 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True)
86 # Fake Butler to avoid Workspace initialization overhead
87 self.setUpMockPatch("lsst.daf.persistence.Butler", autospec=True)
89 self.workspace = WorkspaceGen2(self._testDir)
90 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args(
91 ["--id", "visit=%d" % _getDataIds()[0]["visit"]])
93 @staticmethod
94 def dummyMetadata():
95 result = PropertySet()
96 result.add("lsst.ap.pipe.ccdProcessor.cycleCount", 42)
97 return result
99 def setUpMockPatch(self, target, **kwargs):
100 """Create and register a patcher for a test suite.
102 The patching process is guaranteed to avoid resource leaks or
103 side effects lasting beyond the test case that calls this method.
105 Parameters
106 ----------
107 target : `str`
108 The target to patch. Must obey all restrictions listed
109 for the ``target`` parameter of `unittest.mock.patch`.
110 kwargs : any
111 Any keyword arguments that are allowed for `unittest.mock.patch`,
112 particularly optional attributes for a `unittest.mock.Mock`.
114 Returns
115 -------
116 mock : `unittest.mock.MagicMock`
117 Object representing the same type of entity as ``target``. For
118 example, if ``target`` is the name of a class, this method shall
119 return a replacement class (rather than a replacement object of
120 that class).
121 """
122 patcher = unittest.mock.patch(target, **kwargs)
123 mock = patcher.start()
124 self.addCleanup(patcher.stop)
125 return mock
127 # Mock up ApPipeTask to avoid doing any processing.
128 @patchApPipe
129 def testrunApPipeGen2Steps(self, mockDb, mockClass):
130 """Test that runApPipeGen2 runs the entire pipeline.
131 """
132 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs)
134 mockDb.assert_called_once()
135 mockClass.parseAndRun.assert_called_once()
137 @patchApPipe
138 def testrunApPipeGen2DataIdReporting(self, _mockDb, _mockClass):
139 """Test that runApPipeGen2 reports the data IDs that were processed.
140 """
141 results = pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs)
142 ids = results.parsedCmd.id
144 self.assertEqual(ids.idList, _getDataIds())
146 def _getCmdLineArgs(self, parseAndRunArgs):
147 if parseAndRunArgs[0]:
148 return parseAndRunArgs[0][0]
149 elif "args" in parseAndRunArgs[1]:
150 return parseAndRunArgs[1]["args"]
151 else:
152 self.fail("No command-line args passed to parseAndRun!")
154 @patchApPipe
155 def testrunApPipeGen2CustomConfig(self, _mockDb, mockClass):
156 """Test that runApPipeGen2 can pass custom configs from a workspace to ApPipeTask.
157 """
158 mockParse = mockClass.parseAndRun
159 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs)
160 mockParse.assert_called_once()
161 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args)
162 self.assertIn(os.path.join(self.workspace.configDir, "apPipe.py"), cmdLineArgs)
164 @patchApPipe
165 def testrunApPipeGen2WorkspaceDb(self, mockDb, mockClass):
166 """Test that runApPipeGen2 places a database in the workspace location by default.
167 """
168 mockParse = mockClass.parseAndRun
169 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs)
171 mockDb.assert_called_once()
172 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args)
173 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs)
175 mockParse.assert_called_once()
176 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args)
177 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs)
179 @patchApPipe
180 def testrunApPipeGen2Reuse(self, _mockDb, mockClass):
181 """Test that runApPipeGen2 does not run the pipeline at all (not even with
182 --reuse-outputs-from) if --skip-pipeline is provided.
183 """
184 mockParse = mockClass.parseAndRun
185 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"])
186 pipeline_driver.runApPipeGen2(self.workspace, skipArgs)
187 mockParse.assert_not_called()
190class PipelineDriverTestSuiteGen3(lsst.utils.tests.TestCase):
191 def setUp(self):
192 self._testDir = tempfile.mkdtemp()
193 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True)
195 # Fake Butler to avoid Workspace initialization overhead
196 self.setUpMockPatch("lsst.daf.butler.Registry",
197 **{"queryDatasets.return_value": []})
198 self.setUpMockPatch("lsst.daf.butler.Butler")
200 self.workspace = WorkspaceGen3(self._testDir)
201 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args(
202 ["--id", "visit = %d" % _getDataIds()[0]["visit"]])
204 @staticmethod
205 def dummyMetadata():
206 result = PropertySet()
207 result.add("lsst.pipe.base.calibrate.cycleCount", 42)
208 return result
210 def setUpMockPatch(self, target, **kwargs):
211 """Create and register a patcher for a test suite.
213 The patching process is guaranteed to avoid resource leaks or
214 side effects lasting beyond the test case that calls this method.
216 Parameters
217 ----------
218 target : `str`
219 The target to patch. Must obey all restrictions listed
220 for the ``target`` parameter of `unittest.mock.patch`.
221 kwargs : any
222 Any keyword arguments that are allowed for `unittest.mock.patch`,
223 particularly optional attributes for a `unittest.mock.Mock`.
225 Returns
226 -------
227 mock : `unittest.mock.Mock`
228 Object representing the same type of entity as ``target``. For
229 example, if ``target`` is the name of a class, this method shall
230 return a replacement class (rather than a replacement object of
231 that class).
232 """
233 patcher = unittest.mock.patch(target, **kwargs)
234 mock = patcher.start()
235 self.addCleanup(patcher.stop)
236 return mock
238 # Mock up CmdLineFwk to avoid doing any processing.
239 @patchApPipeGen3
240 def testrunApPipeGen3Steps(self, mockDb, mockFwk):
241 """Test that runApPipeGen3 runs the entire pipeline.
242 """
243 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs)
245 mockDb.assert_called_once()
246 mockFwk().parseAndRun.assert_called_once()
248 def _getCmdLineArgs(self, parseAndRunArgs):
249 if parseAndRunArgs[0]:
250 return parseAndRunArgs[0][0]
251 elif "args" in parseAndRunArgs[1]:
252 return parseAndRunArgs[1]["args"]
253 else:
254 self.fail("No command-line args passed to parseAndRun!")
256 @patchApPipeGen3
257 def testrunApPipeGen3WorkspaceDb(self, mockDb, mockFwk):
258 """Test that runApPipeGen3 places a database in the workspace location by default.
259 """
260 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs)
262 mockDb.assert_called_once()
263 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args)
264 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs)
266 mockParse = mockFwk().parseAndRun
267 mockParse.assert_called_once()
268 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args)
269 self.assertIn("diaPipe:apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs)
271 @patchApPipeGen3
272 def testrunApPipeGen3Reuse(self, _mockDb, mockFwk):
273 """Test that runApPipeGen2 does not run the pipeline at all (not even with
274 --skip-existing) if --skip-pipeline is provided.
275 """
276 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"])
277 pipeline_driver.runApPipeGen3(self.workspace, skipArgs)
278 mockFwk().parseAndRun.assert_not_called()
281class MemoryTester(lsst.utils.tests.MemoryTestCase):
282 pass
285def setup_module(module):
286 lsst.utils.tests.init()
289if __name__ == "__main__": 289 ↛ 290line 289 didn't jump to line 290, because the condition on line 289 was never true
290 lsst.utils.tests.init()
291 unittest.main()