Coverage for tests/test_cmdLineTask.py: 35%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

150 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import shutil 

24import unittest 

25import tempfile 

26 

27import lsst.utils 

28import lsst.pipe.base as pipeBase 

29import lsst.obs.test 

30from lsst.utils.timer import timeMethod 

31import logging 

32 

33ObsTestDir = lsst.utils.getPackageDir("obs_test") 

34DataPath = os.path.join(ObsTestDir, "data", "input") 

35 

36 

37class ExampleTask(pipeBase.CmdLineTask): 

38 ConfigClass = lsst.obs.test.TestConfig 

39 _DefaultName = "test" 

40 

41 def __init__(self, *args, **kwargs): 

42 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

43 self.dataRefList = [] 

44 self.numProcessed = 0 

45 self.metadata["numProcessed"] = self.numProcessed 

46 

47 @timeMethod 

48 def runDataRef(self, dataRef): 

49 if self.config.doFail: 

50 raise pipeBase.TaskError("Failed by request: config.doFail is true") 

51 self.dataRefList.append(dataRef) 

52 self.numProcessed += 1 

53 self.metadata["numProcessed"] = self.numProcessed 

54 return pipeBase.Struct( 

55 numProcessed=self.numProcessed, 

56 ) 

57 

58 

59class CannotConstructTask(ExampleTask): 

60 """A task that cannot be constructed; used to test error handling 

61 """ 

62 

63 def __init__(self, *args, **kwargs): 

64 raise RuntimeError("This task cannot be constructed") 

65 

66 

67class NoMultiprocessTask(ExampleTask): 

68 """Version of ExampleTask that does not support multiprocessing""" 

69 canMultiprocess = False 

70 

71 

72class LegacyTask(ExampleTask): 

73 """Version of ExampleTask with `run` as entry point rather than 

74 `runDataRef` 

75 """ 

76 RunnerClass = pipeBase.LegacyTaskRunner 

77 

78 def run(self, dataRef): 

79 results = self.runDataRef(dataRef) 

80 resultsToBeAdded = pipeBase.Struct(didEnterRun=True) 

81 results.mergeItems(resultsToBeAdded, "didEnterRun") 

82 return results 

83 

84 

85class CmdLineTaskTestCase(unittest.TestCase): 

86 """A test case for CmdLineTask 

87 """ 

88 

89 def setUp(self): 

90 os.environ.pop("PIPE_INPUT_ROOT", None) 

91 os.environ.pop("PIPE_CALIB_ROOT", None) 

92 os.environ.pop("PIPE_OUTPUT_ROOT", None) 

93 self.outPath = tempfile.mkdtemp() 

94 

95 def tearDown(self): 

96 try: 

97 shutil.rmtree(self.outPath) 

98 except Exception: 

99 print("WARNING: failed to remove temporary dir %r" % (self.outPath,)) 

100 del self.outPath 

101 

102 def testBasics(self): 

103 """Test basic construction and use of a command-line task 

104 """ 

105 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=1"]) 

106 self.assertEqual(retVal.resultList, [pipeBase.Struct(exitStatus=0)]) 

107 task = ExampleTask(config=retVal.parsedCmd.config) 

108 parsedCmd = retVal.parsedCmd 

109 self.assertEqual(len(parsedCmd.id.refList), 1) 

110 dataRef = parsedCmd.id.refList[0] 

111 dataId = dataRef.dataId 

112 self.assertEqual(dataId["visit"], 1) 

113 self.assertEqual(task.getName(), "test") 

114 config = dataRef.get("test_config", immediate=True) 

115 self.assertEqual(config, task.config) 

116 metadata = dataRef.get("test_metadata", immediate=True) 

117 self.assertEqual(metadata.getScalar("test.numProcessed"), 1) 

118 

119 def testOverrides(self): 

120 """Test config and log override 

121 """ 

122 config = ExampleTask.ConfigClass() 

123 config.floatField = -99.9 

124 log = logging.getLogger("cmdLineTask") 

125 retVal = ExampleTask.parseAndRun( 

126 args=[DataPath, "--output", self.outPath, "--id", "visit=2"], 

127 config=config, 

128 log=log 

129 ) 

130 self.assertEqual(retVal.parsedCmd.config.floatField, -99.9) 

131 

132 # The logger class may have been changed but the logger name 

133 # should still match. 

134 self.assertEqual(retVal.parsedCmd.log.name, log.name) 

135 

136 def testDoReturnResults(self): 

137 """Test the doReturnResults flag 

138 """ 

139 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

140 "--id", "visit=3", "filter=r"], doReturnResults=True) 

141 self.assertEqual(len(retVal.resultList), 1) 

142 result = retVal.resultList[0] 

143 self.assertEqual(result.metadata.getScalar("numProcessed"), 1) 

144 self.assertEqual(result.result.numProcessed, 1) 

145 

146 def testDoReturnResultsOnFailure(self): 

147 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

148 "--id", "visit=3", "filter=r", "--config", "doFail=True", 

149 "--clobber-config", "--noExit"], doReturnResults=True) 

150 self.assertEqual(len(retVal.resultList), 1) 

151 result = retVal.resultList[0] 

152 self.assertEqual(result.metadata.getScalar("numProcessed"), 0) 

153 self.assertEqual(retVal.resultList[0].result, None) 

154 

155 def testBackupConfig(self): 

156 """Test backup config file creation 

157 """ 

158 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"]) 

159 # Rerun with --clobber-config to ensure backup config file is created 

160 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r", 

161 "--config", "floatField=-99.9", "--clobber-config"]) 

162 # Ensure backup config file was created 

163 self.assertTrue(os.path.exists(os.path.join( 

164 self.outPath, "config", ExampleTask._DefaultName + ".py~1"))) 

165 

166 def testNoBackupConfig(self): 

167 """Test no backup config file creation 

168 """ 

169 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"]) 

170 # Rerun with --clobber-config and --no-backup-config to ensure backup config file is NOT created 

171 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r", 

172 "--config", "floatField=-99.9", "--clobber-config", 

173 "--no-backup-config"]) 

174 # Ensure backup config file was NOT created 

175 self.assertFalse( 

176 os.path.exists(os.path.join(self.outPath, "config", ExampleTask._DefaultName + ".py~1"))) 

177 

178 def testMultiprocess(self): 

179 """Test multiprocessing at a very minimal level 

180 """ 

181 for TaskClass in (ExampleTask, NoMultiprocessTask): 

182 result = TaskClass.parseAndRun(args=[DataPath, "--output", self.outPath, 

183 "-j", "5", "--id", "visit=2", "filter=r"]) 

184 self.assertEqual(result.taskRunner.numProcesses, 5 if TaskClass.canMultiprocess else 1) 

185 

186 def testCannotConstructTask(self): 

187 """Test error handling when a task cannot be constructed 

188 """ 

189 for doRaise in (False, True): 

190 args = [DataPath, "--output", self.outPath, "--id", "visit=1"] 

191 if doRaise: 

192 args.append("--doraise") 

193 with self.assertRaises(RuntimeError): 

194 CannotConstructTask.parseAndRun(args=args) 

195 

196 def testLegacyTask(self): 

197 """Test error handling when a task cannot be constructed 

198 """ 

199 retVal = LegacyTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

200 "--id", "visit=3", "filter=r"], doReturnResults=True) 

201 self.assertEqual(retVal.resultList[0].result.didEnterRun, True) 

202 

203 

204class EaxmpleMultipleIdTaskRunner(pipeBase.TaskRunner): 

205 """TaskRunner to get multiple identifiers down into a Task""" 

206 @staticmethod 

207 def getTargetList(parsedCmd): 

208 """We want our Task to process one dataRef from each identifier at a time""" 

209 return list(zip(parsedCmd.one.refList, parsedCmd.two.refList)) 

210 

211 def __call__(self, target): 

212 """Send results from the Task back so we can inspect 

213 

214 For this test case with obs_test, we know that the results are picklable 

215 and small, so returning something is not a problem. 

216 """ 

217 task = self.TaskClass(config=self.config, log=self.log) 

218 return task.runDataRef(target) 

219 

220 

221class ExampleMultipleIdTask(pipeBase.CmdLineTask): 

222 _DefaultName = "test" 

223 ConfigClass = lsst.obs.test.TestConfig 

224 RunnerClass = EaxmpleMultipleIdTaskRunner 

225 

226 @classmethod 

227 def _makeArgumentParser(cls): 

228 """We want an argument parser that has multiple identifiers""" 

229 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

230 parser.add_id_argument("--one", "raw", "data identifier one", level="sensor") 

231 parser.add_id_argument("--two", "raw", "data identifier two", level="sensor") 

232 return parser 

233 

234 def runDataRef(self, data): 

235 """Our Task just spits back what's in the dataRefs.""" 

236 oneRef = data[0] 

237 twoRef = data[1] 

238 return oneRef.get("raw", snap=0, channel="0,0"), twoRef.get("raw", snap=0, channel="0,0") 

239 

240 

241class MultipleIdTaskTestCase(unittest.TestCase): 

242 """A test case for CmdLineTask using multiple identifiers 

243 

244 Tests implementation of ticket 2144, and demonstrates how 

245 to get results from multiple identifiers down into a Task. 

246 """ 

247 

248 def setUp(self): 

249 os.environ.pop("PIPE_INPUT_ROOT", None) 

250 os.environ.pop("PIPE_CALIB_ROOT", None) 

251 os.environ.pop("PIPE_OUTPUT_ROOT", None) 

252 self.outPath = tempfile.mkdtemp() 

253 

254 def tearDown(self): 

255 try: 

256 shutil.rmtree(self.outPath) 

257 except Exception: 

258 print("WARNING: failed to remove temporary dir %r" % (self.outPath,)) 

259 del self.outPath 

260 

261 def testMultiple(self): 

262 """Test use of a CmdLineTask with multiple identifiers""" 

263 args = [DataPath, "--output", self.outPath, 

264 "--one", "visit=1", "filter=g", 

265 "--two", "visit=2", "filter=g", 

266 ] 

267 retVal = ExampleMultipleIdTask.parseAndRun(args=args) 

268 self.assertEqual(len(retVal.resultList), 1) 

269 

270 

271class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

272 pass 

273 

274 

275def setup_module(module): 

276 lsst.utils.tests.init() 

277 

278 

279if __name__ == "__main__": 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true

280 lsst.utils.tests.init() 

281 unittest.main()