Coverage for tests/test_cmdLineTask.py: 35%

149 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 03:00 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import shutil 

24import unittest 

25import tempfile 

26 

27import lsst.utils 

28import lsst.pipe.base as pipeBase 

29import lsst.obs.test 

30import logging 

31 

32ObsTestDir = lsst.utils.getPackageDir("obs_test") 

33DataPath = os.path.join(ObsTestDir, "data", "input") 

34 

35 

36class ExampleTask(pipeBase.CmdLineTask): 

37 ConfigClass = lsst.obs.test.TestConfig 

38 _DefaultName = "test" 

39 

40 def __init__(self, *args, **kwargs): 

41 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

42 self.dataRefList = [] 

43 self.numProcessed = 0 

44 self.metadata.set("numProcessed", self.numProcessed) 

45 

46 @pipeBase.timeMethod 

47 def runDataRef(self, dataRef): 

48 if self.config.doFail: 

49 raise pipeBase.TaskError("Failed by request: config.doFail is true") 

50 self.dataRefList.append(dataRef) 

51 self.numProcessed += 1 

52 self.metadata.set("numProcessed", self.numProcessed) 

53 return pipeBase.Struct( 

54 numProcessed=self.numProcessed, 

55 ) 

56 

57 

58class CannotConstructTask(ExampleTask): 

59 """A task that cannot be constructed; used to test error handling 

60 """ 

61 

62 def __init__(self, *args, **kwargs): 

63 raise RuntimeError("This task cannot be constructed") 

64 

65 

66class NoMultiprocessTask(ExampleTask): 

67 """Version of ExampleTask that does not support multiprocessing""" 

68 canMultiprocess = False 

69 

70 

71class LegacyTask(ExampleTask): 

72 """Version of ExampleTask with `run` as entry point rather than 

73 `runDataRef` 

74 """ 

75 RunnerClass = pipeBase.LegacyTaskRunner 

76 

77 def run(self, dataRef): 

78 results = self.runDataRef(dataRef) 

79 resultsToBeAdded = pipeBase.Struct(didEnterRun=True) 

80 results.mergeItems(resultsToBeAdded, "didEnterRun") 

81 return results 

82 

83 

84class CmdLineTaskTestCase(unittest.TestCase): 

85 """A test case for CmdLineTask 

86 """ 

87 

88 def setUp(self): 

89 os.environ.pop("PIPE_INPUT_ROOT", None) 

90 os.environ.pop("PIPE_CALIB_ROOT", None) 

91 os.environ.pop("PIPE_OUTPUT_ROOT", None) 

92 self.outPath = tempfile.mkdtemp() 

93 

94 def tearDown(self): 

95 try: 

96 shutil.rmtree(self.outPath) 

97 except Exception: 

98 print("WARNING: failed to remove temporary dir %r" % (self.outPath,)) 

99 del self.outPath 

100 

101 def testBasics(self): 

102 """Test basic construction and use of a command-line task 

103 """ 

104 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=1"]) 

105 self.assertEqual(retVal.resultList, [pipeBase.Struct(exitStatus=0)]) 

106 task = ExampleTask(config=retVal.parsedCmd.config) 

107 parsedCmd = retVal.parsedCmd 

108 self.assertEqual(len(parsedCmd.id.refList), 1) 

109 dataRef = parsedCmd.id.refList[0] 

110 dataId = dataRef.dataId 

111 self.assertEqual(dataId["visit"], 1) 

112 self.assertEqual(task.getName(), "test") 

113 config = dataRef.get("test_config", immediate=True) 

114 self.assertEqual(config, task.config) 

115 metadata = dataRef.get("test_metadata", immediate=True) 

116 self.assertEqual(metadata.getScalar("test.numProcessed"), 1) 

117 

118 def testOverrides(self): 

119 """Test config and log override 

120 """ 

121 config = ExampleTask.ConfigClass() 

122 config.floatField = -99.9 

123 log = logging.getLogger("cmdLineTask") 

124 retVal = ExampleTask.parseAndRun( 

125 args=[DataPath, "--output", self.outPath, "--id", "visit=2"], 

126 config=config, 

127 log=log 

128 ) 

129 self.assertEqual(retVal.parsedCmd.config.floatField, -99.9) 

130 

131 # The logger class may have been changed but the logger name 

132 # should still match. 

133 self.assertEqual(retVal.parsedCmd.log.name, log.name) 

134 

135 def testDoReturnResults(self): 

136 """Test the doReturnResults flag 

137 """ 

138 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

139 "--id", "visit=3", "filter=r"], doReturnResults=True) 

140 self.assertEqual(len(retVal.resultList), 1) 

141 result = retVal.resultList[0] 

142 self.assertEqual(result.metadata.getScalar("numProcessed"), 1) 

143 self.assertEqual(result.result.numProcessed, 1) 

144 

145 def testDoReturnResultsOnFailure(self): 

146 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

147 "--id", "visit=3", "filter=r", "--config", "doFail=True", 

148 "--clobber-config", "--noExit"], doReturnResults=True) 

149 self.assertEqual(len(retVal.resultList), 1) 

150 result = retVal.resultList[0] 

151 self.assertEqual(result.metadata.getScalar("numProcessed"), 0) 

152 self.assertEqual(retVal.resultList[0].result, None) 

153 

154 def testBackupConfig(self): 

155 """Test backup config file creation 

156 """ 

157 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"]) 

158 # Rerun with --clobber-config to ensure backup config file is created 

159 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r", 

160 "--config", "floatField=-99.9", "--clobber-config"]) 

161 # Ensure backup config file was created 

162 self.assertTrue(os.path.exists(os.path.join( 

163 self.outPath, "config", ExampleTask._DefaultName + ".py~1"))) 

164 

165 def testNoBackupConfig(self): 

166 """Test no backup config file creation 

167 """ 

168 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"]) 

169 # Rerun with --clobber-config and --no-backup-config to ensure backup config file is NOT created 

170 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r", 

171 "--config", "floatField=-99.9", "--clobber-config", 

172 "--no-backup-config"]) 

173 # Ensure backup config file was NOT created 

174 self.assertFalse( 

175 os.path.exists(os.path.join(self.outPath, "config", ExampleTask._DefaultName + ".py~1"))) 

176 

177 def testMultiprocess(self): 

178 """Test multiprocessing at a very minimal level 

179 """ 

180 for TaskClass in (ExampleTask, NoMultiprocessTask): 

181 result = TaskClass.parseAndRun(args=[DataPath, "--output", self.outPath, 

182 "-j", "5", "--id", "visit=2", "filter=r"]) 

183 self.assertEqual(result.taskRunner.numProcesses, 5 if TaskClass.canMultiprocess else 1) 

184 

185 def testCannotConstructTask(self): 

186 """Test error handling when a task cannot be constructed 

187 """ 

188 for doRaise in (False, True): 

189 args = [DataPath, "--output", self.outPath, "--id", "visit=1"] 

190 if doRaise: 

191 args.append("--doraise") 

192 with self.assertRaises(RuntimeError): 

193 CannotConstructTask.parseAndRun(args=args) 

194 

195 def testLegacyTask(self): 

196 """Test error handling when a task cannot be constructed 

197 """ 

198 retVal = LegacyTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

199 "--id", "visit=3", "filter=r"], doReturnResults=True) 

200 self.assertEqual(retVal.resultList[0].result.didEnterRun, True) 

201 

202 

203class EaxmpleMultipleIdTaskRunner(pipeBase.TaskRunner): 

204 """TaskRunner to get multiple identifiers down into a Task""" 

205 @staticmethod 

206 def getTargetList(parsedCmd): 

207 """We want our Task to process one dataRef from each identifier at a time""" 

208 return list(zip(parsedCmd.one.refList, parsedCmd.two.refList)) 

209 

210 def __call__(self, target): 

211 """Send results from the Task back so we can inspect 

212 

213 For this test case with obs_test, we know that the results are picklable 

214 and small, so returning something is not a problem. 

215 """ 

216 task = self.TaskClass(config=self.config, log=self.log) 

217 return task.runDataRef(target) 

218 

219 

220class ExampleMultipleIdTask(pipeBase.CmdLineTask): 

221 _DefaultName = "test" 

222 ConfigClass = lsst.obs.test.TestConfig 

223 RunnerClass = EaxmpleMultipleIdTaskRunner 

224 

225 @classmethod 

226 def _makeArgumentParser(cls): 

227 """We want an argument parser that has multiple identifiers""" 

228 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

229 parser.add_id_argument("--one", "raw", "data identifier one", level="sensor") 

230 parser.add_id_argument("--two", "raw", "data identifier two", level="sensor") 

231 return parser 

232 

233 def runDataRef(self, data): 

234 """Our Task just spits back what's in the dataRefs.""" 

235 oneRef = data[0] 

236 twoRef = data[1] 

237 return oneRef.get("raw", snap=0, channel="0,0"), twoRef.get("raw", snap=0, channel="0,0") 

238 

239 

240class MultipleIdTaskTestCase(unittest.TestCase): 

241 """A test case for CmdLineTask using multiple identifiers 

242 

243 Tests implementation of ticket 2144, and demonstrates how 

244 to get results from multiple identifiers down into a Task. 

245 """ 

246 

247 def setUp(self): 

248 os.environ.pop("PIPE_INPUT_ROOT", None) 

249 os.environ.pop("PIPE_CALIB_ROOT", None) 

250 os.environ.pop("PIPE_OUTPUT_ROOT", None) 

251 self.outPath = tempfile.mkdtemp() 

252 

253 def tearDown(self): 

254 try: 

255 shutil.rmtree(self.outPath) 

256 except Exception: 

257 print("WARNING: failed to remove temporary dir %r" % (self.outPath,)) 

258 del self.outPath 

259 

260 def testMultiple(self): 

261 """Test use of a CmdLineTask with multiple identifiers""" 

262 args = [DataPath, "--output", self.outPath, 

263 "--one", "visit=1", "filter=g", 

264 "--two", "visit=2", "filter=g", 

265 ] 

266 retVal = ExampleMultipleIdTask.parseAndRun(args=args) 

267 self.assertEqual(len(retVal.resultList), 1) 

268 

269 

270class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

271 pass 

272 

273 

274def setup_module(module): 

275 lsst.utils.tests.init() 

276 

277 

278if __name__ == "__main__": 278 ↛ 279line 278 didn't jump to line 279, because the condition on line 278 was never true

279 lsst.utils.tests.init() 

280 unittest.main()