Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import argparse 

25import functools 

26import os 

27import shutil 

28import tempfile 

29import unittest.mock 

30 

31from lsst.daf.base import PropertySet 

32from lsst.pipe.base import DataIdContainer, Struct 

33import lsst.utils.tests 

34from lsst.ap.pipe import ApPipeTask 

35from lsst.ap.verify import pipeline_driver 

36from lsst.ap.verify.workspace import WorkspaceGen2, WorkspaceGen3 

37 

38 

39def _getDataIds(): 

40 return [{"visit": 42, "ccd": 0}] 

41 

42 

43def patchApPipe(method): 

44 """Shortcut decorator for consistently patching ApPipeTask. 

45 """ 

46 @functools.wraps(method) 

47 def wrapper(self, *args, **kwargs): 

48 parsedCmd = argparse.Namespace() 

49 parsedCmd.id = DataIdContainer() 

50 parsedCmd.id.idList = _getDataIds() 

51 parReturn = Struct( 

52 argumentParser=None, 

53 parsedCmd=parsedCmd, 

54 taskRunner=None, 

55 resultList=[Struct(exitStatus=0)]) 

56 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb") 

57 pipePatcher = unittest.mock.patch("lsst.ap.pipe.ApPipeTask", 

58 **{"parseAndRun.return_value": parReturn}, 

59 _DefaultName=ApPipeTask._DefaultName, 

60 ConfigClass=ApPipeTask.ConfigClass) 

61 patchedMethod = pipePatcher(dbPatcher(method)) 

62 return patchedMethod(self, *args, **kwargs) 

63 return wrapper 

64 

65 

66def patchApPipeGen3(method): 

67 """Shortcut decorator for consistently patching AP code. 

68 """ 

69 @functools.wraps(method) 

70 def wrapper(self, *args, **kwargs): 

71 parsedCmd = argparse.Namespace() 

72 parsedCmd.id = DataIdContainer() 

73 parsedCmd.id.idList = _getDataIds() 

74 dbPatcher = unittest.mock.patch("lsst.ap.verify.pipeline_driver.makeApdb") 

75 execPatcher = unittest.mock.patch("lsst.ctrl.mpexec.CmdLineFwk") 

76 patchedMethod = execPatcher(dbPatcher(method)) 

77 return patchedMethod(self, *args, **kwargs) 

78 return wrapper 

79 

80 

81class PipelineDriverTestSuiteGen2(lsst.utils.tests.TestCase): 

82 def setUp(self): 

83 self._testDir = tempfile.mkdtemp() 

84 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True) 

85 

86 # Fake Butler to avoid Workspace initialization overhead 

87 self.setUpMockPatch("lsst.daf.persistence.Butler", autospec=True) 

88 

89 self.workspace = WorkspaceGen2(self._testDir) 

90 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args( 

91 ["--id", "visit=%d" % _getDataIds()[0]["visit"]]) 

92 

93 @staticmethod 

94 def dummyMetadata(): 

95 result = PropertySet() 

96 result.add("lsst.ap.pipe.ccdProcessor.cycleCount", 42) 

97 return result 

98 

99 def setUpMockPatch(self, target, **kwargs): 

100 """Create and register a patcher for a test suite. 

101 

102 The patching process is guaranteed to avoid resource leaks or 

103 side effects lasting beyond the test case that calls this method. 

104 

105 Parameters 

106 ---------- 

107 target : `str` 

108 The target to patch. Must obey all restrictions listed 

109 for the ``target`` parameter of `unittest.mock.patch`. 

110 kwargs : any 

111 Any keyword arguments that are allowed for `unittest.mock.patch`, 

112 particularly optional attributes for a `unittest.mock.Mock`. 

113 

114 Returns 

115 ------- 

116 mock : `unittest.mock.MagicMock` 

117 Object representing the same type of entity as ``target``. For 

118 example, if ``target`` is the name of a class, this method shall 

119 return a replacement class (rather than a replacement object of 

120 that class). 

121 """ 

122 patcher = unittest.mock.patch(target, **kwargs) 

123 mock = patcher.start() 

124 self.addCleanup(patcher.stop) 

125 return mock 

126 

127 # Mock up ApPipeTask to avoid doing any processing. 

128 @patchApPipe 

129 def testrunApPipeGen2Steps(self, mockDb, mockClass): 

130 """Test that runApPipeGen2 runs the entire pipeline. 

131 """ 

132 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

133 

134 mockDb.assert_called_once() 

135 mockClass.parseAndRun.assert_called_once() 

136 

137 @patchApPipe 

138 def testrunApPipeGen2DataIdReporting(self, _mockDb, _mockClass): 

139 """Test that runApPipeGen2 reports the data IDs that were processed. 

140 """ 

141 results = pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

142 ids = results.parsedCmd.id 

143 

144 self.assertEqual(ids.idList, _getDataIds()) 

145 

146 def _getCmdLineArgs(self, parseAndRunArgs): 

147 if parseAndRunArgs[0]: 

148 return parseAndRunArgs[0][0] 

149 elif "args" in parseAndRunArgs[1]: 

150 return parseAndRunArgs[1]["args"] 

151 else: 

152 self.fail("No command-line args passed to parseAndRun!") 

153 

154 @patchApPipe 

155 def testrunApPipeGen2CustomConfig(self, _mockDb, mockClass): 

156 """Test that runApPipeGen2 can pass custom configs from a workspace to ApPipeTask. 

157 """ 

158 mockParse = mockClass.parseAndRun 

159 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

160 mockParse.assert_called_once() 

161 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

162 self.assertIn(os.path.join(self.workspace.configDir, "apPipe.py"), cmdLineArgs) 

163 

164 @patchApPipe 

165 def testrunApPipeGen2WorkspaceDb(self, mockDb, mockClass): 

166 """Test that runApPipeGen2 places a database in the workspace location by default. 

167 """ 

168 mockParse = mockClass.parseAndRun 

169 pipeline_driver.runApPipeGen2(self.workspace, self.apPipeArgs) 

170 

171 mockDb.assert_called_once() 

172 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

173 self.assertIn("db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

174 

175 mockParse.assert_called_once() 

176 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

177 self.assertIn("diaPipe.apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

178 

179 @patchApPipe 

180 def testrunApPipeGen2Reuse(self, _mockDb, mockClass): 

181 """Test that runApPipeGen2 does not run the pipeline at all (not even with 

182 --reuse-outputs-from) if --skip-pipeline is provided. 

183 """ 

184 mockParse = mockClass.parseAndRun 

185 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"]) 

186 pipeline_driver.runApPipeGen2(self.workspace, skipArgs) 

187 mockParse.assert_not_called() 

188 

189 

190class PipelineDriverTestSuiteGen3(lsst.utils.tests.TestCase): 

191 def setUp(self): 

192 self._testDir = tempfile.mkdtemp() 

193 self.addCleanup(shutil.rmtree, self._testDir, ignore_errors=True) 

194 

195 # Fake Butler to avoid Workspace initialization overhead 

196 self.setUpMockPatch("lsst.daf.butler.Registry", 

197 **{"queryDatasets.return_value": []}) 

198 self.setUpMockPatch("lsst.daf.butler.Butler") 

199 

200 self.workspace = WorkspaceGen3(self._testDir) 

201 self.apPipeArgs = pipeline_driver.ApPipeParser().parse_args( 

202 ["--id", "visit = %d" % _getDataIds()[0]["visit"]]) 

203 

204 @staticmethod 

205 def dummyMetadata(): 

206 result = PropertySet() 

207 result.add("lsst.pipe.base.calibrate.cycleCount", 42) 

208 return result 

209 

210 def setUpMockPatch(self, target, **kwargs): 

211 """Create and register a patcher for a test suite. 

212 

213 The patching process is guaranteed to avoid resource leaks or 

214 side effects lasting beyond the test case that calls this method. 

215 

216 Parameters 

217 ---------- 

218 target : `str` 

219 The target to patch. Must obey all restrictions listed 

220 for the ``target`` parameter of `unittest.mock.patch`. 

221 kwargs : any 

222 Any keyword arguments that are allowed for `unittest.mock.patch`, 

223 particularly optional attributes for a `unittest.mock.Mock`. 

224 

225 Returns 

226 ------- 

227 mock : `unittest.mock.Mock` 

228 Object representing the same type of entity as ``target``. For 

229 example, if ``target`` is the name of a class, this method shall 

230 return a replacement class (rather than a replacement object of 

231 that class). 

232 """ 

233 patcher = unittest.mock.patch(target, **kwargs) 

234 mock = patcher.start() 

235 self.addCleanup(patcher.stop) 

236 return mock 

237 

238 @unittest.skip("Fix test in DM-27117") 

239 # Mock up CmdLineFwk to avoid doing any processing. 

240 @patchApPipeGen3 

241 def testrunApPipeGen3Steps(self, mockDb, mockFwk): 

242 """Test that runApPipeGen3 runs the entire pipeline. 

243 """ 

244 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

245 

246 mockDb.assert_called_once() 

247 mockFwk().parseAndRun.assert_called_once() 

248 

249 def _getCmdLineArgs(self, parseAndRunArgs): 

250 if parseAndRunArgs[0]: 

251 return parseAndRunArgs[0][0] 

252 elif "args" in parseAndRunArgs[1]: 

253 return parseAndRunArgs[1]["args"] 

254 else: 

255 self.fail("No command-line args passed to parseAndRun!") 

256 

257 @unittest.skip("Fix test in DM-27117") 

258 @patchApPipeGen3 

259 def testrunApPipeGen3WorkspaceDb(self, mockDb, mockFwk): 

260 """Test that runApPipeGen3 places a database in the workspace location by default. 

261 """ 

262 pipeline_driver.runApPipeGen3(self.workspace, self.apPipeArgs) 

263 

264 mockDb.assert_called_once() 

265 cmdLineArgs = self._getCmdLineArgs(mockDb.call_args) 

266 self.assertIn("db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

267 

268 mockParse = mockFwk().parseAndRun 

269 mockParse.assert_called_once() 

270 cmdLineArgs = self._getCmdLineArgs(mockParse.call_args) 

271 self.assertIn("diaPipe:apdb.db_url=sqlite:///" + self.workspace.dbLocation, cmdLineArgs) 

272 

273 @unittest.skip("Fix test in DM-27117") 

274 @patchApPipeGen3 

275 def testrunApPipeGen3Reuse(self, _mockDb, mockFwk): 

276 """Test that runApPipeGen2 does not run the pipeline at all (not even with 

277 --skip-existing) if --skip-pipeline is provided. 

278 """ 

279 skipArgs = pipeline_driver.ApPipeParser().parse_args(["--skip-pipeline"]) 

280 pipeline_driver.runApPipeGen3(self.workspace, skipArgs) 

281 mockFwk().parseAndRun.assert_not_called() 

282 

283 

284class MemoryTester(lsst.utils.tests.MemoryTestCase): 

285 pass 

286 

287 

288def setup_module(module): 

289 lsst.utils.tests.init() 

290 

291 

292if __name__ == "__main__": 292 ↛ 293line 292 didn't jump to line 293, because the condition on line 292 was never true

293 lsst.utils.tests.init() 

294 unittest.main()