Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import argparse 

25import functools 

26import os 

27import shutil 

28import tempfile 

29import unittest.mock 

30 

31from lsst.daf.base import PropertySet 

32from lsst.pipe.base import DataIdContainer, Struct 

33import lsst.utils.tests 

34from lsst.ap.pipe import ApPipeTask 

35from lsst.ap.verify import pipeline_driver 

36from lsst.ap.verify.workspace import WorkspaceGen2 

37 

38 

39def _getDataIds(): 

40 return [{"visit": 42, "ccd": 0}] 

41 

42 

43def patchApPipe(method): 

44 """Shortcut decorator for consistently patching ApPipeTask. 

45 """ 

46 @functools.wraps(method) 

47 def wrapper(self, *args, **kwargs): 

48 parsedCmd = argparse.Namespace() 

49 parsedCmd.id = DataIdContainer() 

50 parsedCmd.id.idList = _getDataIds() 

51 parReturn = Struct( 

52 argumentParser=None, 

53 parsedCmd=parsedCmd, 

54 taskRunner=None, 

55 resultList=[Struct(exitStatus=0)]) 

56 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb") 

57 pipePatcher = unittest.mock.patch("lsst.ap.pipe.ApPipeTask", 

58 **{"parseAndRun.return_value": parReturn}, 

59 _DefaultName=ApPipeTask._DefaultName, 

60 ConfigClass=ApPipeTask.ConfigClass) 

61 patchedMethod = pipePatcher(dbPatcher(method)) 

62 return patchedMethod(self, *args, **kwargs) 

63 return wrapper 

64 

65 

66class PipelineDriverTestSuite(lsst.utils.tests.TestCase): 

67 def setUp(self): 

68 self._testDir = tempfile.mkdtemp() 

69 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True) 

70 

71 # Fake Butler to avoid Workspace initialization overhead 

72 self.setUpMockPatch("lsst.daf.persistence.Butler", autospec=True) 

73 

74 self.workspace = WorkspaceGen2(self._testDir) 

75 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args( 

76 ["--id", "visit=%d" % _getDataIds()[0]["visit"]]) 

77 

78 @staticmethod 

79 def dummyMetadata(): 

80 result = PropertySet() 

81 result.add("lsst.ap.pipe.ccdProcessor.cycleCount", 42) 

82 return result 

83 

84 def setUpMockPatch(self, target, **kwargs): 

85 """Create and register a patcher for a test suite. 

86 

87 The patching process is guaranteed to avoid resource leaks or 

88 side effects lasting beyond the test case that calls this method. 

89 

90 Parameters 

91 ---------- 

92 target : `str` 

93 The target to patch. Must obey all restrictions listed 

94 for the ``target`` parameter of `unittest.mock.patch`. 

95 kwargs : any 

96 Any keyword arguments that are allowed for `unittest.mock.patch`, 

97 particularly optional attributes for a `unittest.mock.Mock`. 

98 

99 Returns 

100 ------- 

101 mock : `unittest.mock.MagicMock` 

102 Object representing the same type of entity as ``target``. For 

103 example, if ``target`` is the name of a class, this method shall 

104 return a replacement class (rather than a replacement object of 

105 that class). 

106 """ 

107 patcher = unittest.mock.patch(target, **kwargs) 

108 mock = patcher.start() 

109 self.addCleanup(patcher.stop) 

110 return mock 

111 

112 # Mock up ApPipeTask to avoid doing any processing. 

113 @patchApPipe 

114 def testRunApPipeSteps(self, mockDb, mockClass): 

115 """Test that runApPipe runs the entire pipeline. 

116 """ 

117 pipeline_driver.runApPipe(self.workspace, self.apPipeArgs) 

118 

119 mockDb.assert_called_once() 

120 mockClass.parseAndRun.assert_called_once() 

121 

122 @patchApPipe 

123 def testRunApPipeDataIdReporting(self, _mockDb, _mockClass): 

124 """Test that runApPipe reports the data IDs that were processed. 

125 """ 

126 results = pipeline_driver.runApPipe(self.workspace, self.apPipeArgs) 

127 ids = results.parsedCmd.id 

128 

129 self.assertEqual(ids.idList, _getDataIds()) 

130 

131 def _getCmdLineArgs(self, parseAndRunArgs): 

132 if parseAndRunArgs[0]: 

133 return parseAndRunArgs[0][0] 

134 elif "args" in parseAndRunArgs[1]: 

135 return parseAndRunArgs[1]["args"] 

136 else: 

137 self.fail("No command-line args passed to parseAndRun!") 

138 

139 @patchApPipe 

140 def testRunApPipeCustomConfig(self, _mockDb, mockClass): 

141 """Test that runApPipe can pass custom configs from a workspace to ApPipeTask. 

142 """ 

143 mockParse = mockClass.parseAndRun 

144 pipeline_driver.runApPipe(self.workspace, self.apPipeArgs) 

145 mockParse.assert_called_once() 

146 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

147 self.assertIn(os.path.join(self.workspace.configDir, "apPipe.py"), cmdLineArgs) 

148 

149 @patchApPipe 

150 def testRunApPipeWorkspaceDb(self, mockDb, mockClass): 

151 """Test that runApPipe places a database in the workspace location by default. 

152 """ 

153 mockParse = mockClass.parseAndRun 

154 pipeline_driver.runApPipe(self.workspace, self.apPipeArgs) 

155 

156 mockDb.assert_called_once() 

157 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

158 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

159 

160 mockParse.assert_called_once() 

161 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

162 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

163 

164 @patchApPipe 

165 def testRunApPipeReuse(self, _mockDb, mockClass): 

166 """Test that runApPipe does not run the pipeline at all (not even with 

167 --reuse-outputs-from) if --skip-pipeline is provided. 

168 """ 

169 mockParse = mockClass.parseAndRun 

170 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"]) 

171 pipeline_driver.runApPipe(self.workspace, skipArgs) 

172 mockParse.assert_not_called() 

173 

174 

175class MemoryTester(lsst.utils.tests.MemoryTestCase): 

176 pass 

177 

178 

179def setup_module(module): 

180 lsst.utils.tests.init() 

181 

182 

183if __name__ == "__main__": 183 ↛ 184line 183 didn't jump to line 184, because the condition on line 183 was never true

184 lsst.utils.tests.init() 

185 unittest.main()