Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import argparse 

25import functools 

26import os 

27import shutil 

28import tempfile 

29import unittest.mock 

30 

31from lsst.daf.base import PropertySet 

32from lsst.pipe.base import DataIdContainer, Struct 

33import lsst.utils.tests 

34from lsst.ap.pipe import ApPipeTask 

35from lsst.ap.verify import pipeline_driver 

36from lsst.ap.verify.workspace import WorkspaceGen2, WorkspaceGen3 

37 

38 

39def _getDataIds(): 

40 return [{"visit": 42, "ccd": 0}] 

41 

42 

43def patchApPipe(method): 

44 """Shortcut decorator for consistently patching ApPipeTask. 

45 """ 

46 @functools.wraps(method) 

47 def wrapper(self, *args, **kwargs): 

48 parsedCmd = argparse.Namespace() 

49 parsedCmd.id = DataIdContainer() 

50 parsedCmd.id.idList = _getDataIds() 

51 parReturn = Struct( 

52 argumentParser=None, 

53 parsedCmd=parsedCmd, 

54 taskRunner=None, 

55 resultList=[Struct(exitStatus=0)]) 

56 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb") 

57 pipePatcher = unittest.mock.patch("lsst.ap.pipe.ApPipeTask", 

58 **{"parseAndRun.return_value": parReturn}, 

59 _DefaultName=ApPipeTask._DefaultName, 

60 ConfigClass=ApPipeTask.ConfigClass) 

61 patchedMethod = pipePatcher(dbPatcher(method)) 

62 return patchedMethod(self, *args, **kwargs) 

63 return wrapper 

64 

65 

66def patchApPipeGen3(method): 

67 """Shortcut decorator for consistently patching AP code. 

68 """ 

69 @functools.wraps(method) 

70 def wrapper(self, *args, **kwargs): 

71 parsedCmd = argparse.Namespace() 

72 parsedCmd.id = DataIdContainer() 

73 parsedCmd.id.idList = _getDataIds() 

74 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb") 

75 execPatcher = unittest.mock.patch("lsst.ctrl.mpexec.CmdLineFwk") 

76 patchedMethod = execPatcher(dbPatcher(method)) 

77 return patchedMethod(self, *args, **kwargs) 

78 return wrapper 

79 

80 

81class PipelineDriverTestSuiteGen2(lsst.utils.tests.TestCase): 

82 def setUp(self): 

83 self._testDir = tempfile.mkdtemp() 

84 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True) 

85 

86 # Fake Butler to avoid Workspace initialization overhead 

87 self.setUpMockPatch("lsst.daf.persistence.Butler", autospec=True) 

88 

89 self.workspace = WorkspaceGen2(self._testDir) 

90 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args( 

91 ["--id", "visit=%d" % _getDataIds()[0]["visit"]]) 

92 

93 @staticmethod 

94 def dummyMetadata(): 

95 result = PropertySet() 

96 result.add("lsst.ap.pipe.ccdProcessor.cycleCount", 42) 

97 return result 

98 

99 def setUpMockPatch(self, target, **kwargs): 

100 """Create and register a patcher for a test suite. 

101 

102 The patching process is guaranteed to avoid resource leaks or 

103 side effects lasting beyond the test case that calls this method. 

104 

105 Parameters 

106 ---------- 

107 target : `str` 

108 The target to patch. Must obey all restrictions listed 

109 for the ``target`` parameter of `unittest.mock.patch`. 

110 kwargs : any 

111 Any keyword arguments that are allowed for `unittest.mock.patch`, 

112 particularly optional attributes for a `unittest.mock.Mock`. 

113 

114 Returns 

115 ------- 

116 mock : `unittest.mock.MagicMock` 

117 Object representing the same type of entity as ``target``. For 

118 example, if ``target`` is the name of a class, this method shall 

119 return a replacement class (rather than a replacement object of 

120 that class). 

121 """ 

122 patcher = unittest.mock.patch(target, **kwargs) 

123 mock = patcher.start() 

124 self.addCleanup(patcher.stop) 

125 return mock 

126 

127 # Mock up ApPipeTask to avoid doing any processing. 

128 @patchApPipe 

129 def testrunApPipeGen2Steps(self, mockDb, mockClass): 

130 """Test that runApPipeGen2 runs the entire pipeline. 

131 """ 

132 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

133 

134 mockDb.assert_called_once() 

135 mockClass.parseAndRun.assert_called_once() 

136 

137 @patchApPipe 

138 def testrunApPipeGen2DataIdReporting(self, _mockDb, _mockClass): 

139 """Test that runApPipeGen2 reports the data IDs that were processed. 

140 """ 

141 results = pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

142 ids = results.parsedCmd.id 

143 

144 self.assertEqual(ids.idList, _getDataIds()) 

145 

146 def _getCmdLineArgs(self, parseAndRunArgs): 

147 if parseAndRunArgs[0]: 

148 return parseAndRunArgs[0][0] 

149 elif "args" in parseAndRunArgs[1]: 

150 return parseAndRunArgs[1]["args"] 

151 else: 

152 self.fail("No command-line args passed to parseAndRun!") 

153 

154 @patchApPipe 

155 def testrunApPipeGen2CustomConfig(self, _mockDb, mockClass): 

156 """Test that runApPipeGen2 can pass custom configs from a workspace to ApPipeTask. 

157 """ 

158 mockParse = mockClass.parseAndRun 

159 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

160 mockParse.assert_called_once() 

161 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

162 self.assertIn(os.path.join(self.workspace.configDir, "apPipe.py"), cmdLineArgs) 

163 

164 @patchApPipe 

165 def testrunApPipeGen2WorkspaceDb(self, mockDb, mockClass): 

166 """Test that runApPipeGen2 places a database in the workspace location by default. 

167 """ 

168 mockParse = mockClass.parseAndRun 

169 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

170 

171 mockDb.assert_called_once() 

172 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

173 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

174 

175 mockParse.assert_called_once() 

176 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

177 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

178 

179 @patchApPipe 

180 def testrunApPipeGen2Reuse(self, _mockDb, mockClass): 

181 """Test that runApPipeGen2 does not run the pipeline at all (not even with 

182 --reuse-outputs-from) if --skip-pipeline is provided. 

183 """ 

184 mockParse = mockClass.parseAndRun 

185 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"]) 

186 pipeline_driver.runApPipeGen2(self.workspace, skipArgs) 

187 mockParse.assert_not_called() 

188 

189 

190class PipelineDriverTestSuiteGen3(lsst.utils.tests.TestCase): 

191 def setUp(self): 

192 self._testDir = tempfile.mkdtemp() 

193 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True) 

194 

195 # Fake Butler to avoid Workspace initialization overhead 

196 self.setUpMockPatch("lsst.daf.butler.Registry", 

197 **{"queryDatasets.return_value": []}) 

198 self.setUpMockPatch("lsst.daf.butler.Butler") 

199 

200 self.workspace = WorkspaceGen3(self._testDir) 

201 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args( 

202 ["--id", "visit = %d" % _getDataIds()[0]["visit"]]) 

203 

204 @staticmethod 

205 def dummyMetadata(): 

206 result = PropertySet() 

207 result.add("lsst.pipe.base.calibrate.cycleCount", 42) 

208 return result 

209 

210 def setUpMockPatch(self, target, **kwargs): 

211 """Create and register a patcher for a test suite. 

212 

213 The patching process is guaranteed to avoid resource leaks or 

214 side effects lasting beyond the test case that calls this method. 

215 

216 Parameters 

217 ---------- 

218 target : `str` 

219 The target to patch. Must obey all restrictions listed 

220 for the ``target`` parameter of `unittest.mock.patch`. 

221 kwargs : any 

222 Any keyword arguments that are allowed for `unittest.mock.patch`, 

223 particularly optional attributes for a `unittest.mock.Mock`. 

224 

225 Returns 

226 ------- 

227 mock : `unittest.mock.Mock` 

228 Object representing the same type of entity as ``target``. For 

229 example, if ``target`` is the name of a class, this method shall 

230 return a replacement class (rather than a replacement object of 

231 that class). 

232 """ 

233 patcher = unittest.mock.patch(target, **kwargs) 

234 mock = patcher.start() 

235 self.addCleanup(patcher.stop) 

236 return mock 

237 

238 # Mock up CmdLineFwk to avoid doing any processing. 

239 @patchApPipeGen3 

240 def testrunApPipeGen3Steps(self, mockDb, mockFwk): 

241 """Test that runApPipeGen3 runs the entire pipeline. 

242 """ 

243 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

244 

245 mockDb.assert_called_once() 

246 mockFwk().parseAndRun.assert_called_once() 

247 

248 def _getCmdLineArgs(self, parseAndRunArgs): 

249 if parseAndRunArgs[0]: 

250 return parseAndRunArgs[0][0] 

251 elif "args" in parseAndRunArgs[1]: 

252 return parseAndRunArgs[1]["args"] 

253 else: 

254 self.fail("No command-line args passed to parseAndRun!") 

255 

256 @patchApPipeGen3 

257 def testrunApPipeGen3WorkspaceDb(self, mockDb, mockFwk): 

258 """Test that runApPipeGen3 places a database in the workspace location by default. 

259 """ 

260 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

261 

262 mockDb.assert_called_once() 

263 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

264 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

265 

266 mockParse = mockFwk().parseAndRun 

267 mockParse.assert_called_once() 

268 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

269 self.assertIn("diaPipe:apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

270 

271 @patchApPipeGen3 

272 def testrunApPipeGen3Reuse(self, _mockDb, mockFwk): 

273 """Test that runApPipeGen2 does not run the pipeline at all (not even with 

274 --skip-existing) if --skip-pipeline is provided. 

275 """ 

276 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"]) 

277 pipeline_driver.runApPipeGen3(self.workspace, skipArgs) 

278 mockFwk().parseAndRun.assert_not_called() 

279 

280 

281class MemoryTester(lsst.utils.tests.MemoryTestCase): 

282 pass 

283 

284 

285def setup_module(module): 

286 lsst.utils.tests.init() 

287 

288 

289if __name__ == "__main__": 289 ↛ 290line 289 didn't jump to line 290, because the condition on line 289 was never true

290 lsst.utils.tests.init() 

291 unittest.main()