Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import shutil 

24import unittest 

25import tempfile 

26 

27import lsst.utils 

28import lsst.pipe.base as pipeBase 

29import lsst.obs.test 

30from lsst.log import Log 

31 

32ObsTestDir = lsst.utils.getPackageDir("obs_test") 

33DataPath = os.path.join(ObsTestDir, "data", "input") 

34 

35 

36class ExampleTask(pipeBase.CmdLineTask): 

37 ConfigClass = lsst.obs.test.TestConfig 

38 _DefaultName = "test" 

39 

40 def __init__(self, *args, **kwargs): 

41 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

42 self.dataRefList = [] 

43 self.numProcessed = 0 

44 self.metadata.set("numProcessed", self.numProcessed) 

45 

46 @pipeBase.timeMethod 

47 def runDataRef(self, dataRef): 

48 if self.config.doFail: 

49 raise pipeBase.TaskError("Failed by request: config.doFail is true") 

50 self.dataRefList.append(dataRef) 

51 self.numProcessed += 1 

52 self.metadata.set("numProcessed", self.numProcessed) 

53 return pipeBase.Struct( 

54 numProcessed=self.numProcessed, 

55 ) 

56 

57 

58class CannotConstructTask(ExampleTask): 

59 """A task that cannot be constructed; used to test error handling 

60 """ 

61 

62 def __init__(self, *args, **kwargs): 

63 raise RuntimeError("This task cannot be constructed") 

64 

65 

66class NoMultiprocessTask(ExampleTask): 

67 """Version of ExampleTask that does not support multiprocessing""" 

68 canMultiprocess = False 

69 

70 

71class LegacyTask(ExampleTask): 

72 """Version of ExampleTask with `run` as entry point rather than 

73 `runDataRef` 

74 """ 

75 RunnerClass = pipeBase.LegacyTaskRunner 

76 

77 def run(self, dataRef): 

78 results = self.runDataRef(dataRef) 

79 resultsToBeAdded = pipeBase.Struct(didEnterRun=True) 

80 results.mergeItems(resultsToBeAdded, "didEnterRun") 

81 return results 

82 

83 

84class CmdLineTaskTestCase(unittest.TestCase): 

85 """A test case for CmdLineTask 

86 """ 

87 

88 def setUp(self): 

89 os.environ.pop("PIPE_INPUT_ROOT", None) 

90 os.environ.pop("PIPE_CALIB_ROOT", None) 

91 os.environ.pop("PIPE_OUTPUT_ROOT", None) 

92 self.outPath = tempfile.mkdtemp() 

93 

94 def tearDown(self): 

95 try: 

96 shutil.rmtree(self.outPath) 

97 except Exception: 

98 print("WARNING: failed to remove temporary dir %r" % (self.outPath,)) 

99 del self.outPath 

100 

101 def testBasics(self): 

102 """Test basic construction and use of a command-line task 

103 """ 

104 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=1"]) 

105 self.assertEqual(retVal.resultList, [pipeBase.Struct(exitStatus=0)]) 

106 task = ExampleTask(config=retVal.parsedCmd.config) 

107 parsedCmd = retVal.parsedCmd 

108 self.assertEqual(len(parsedCmd.id.refList), 1) 

109 dataRef = parsedCmd.id.refList[0] 

110 dataId = dataRef.dataId 

111 self.assertEqual(dataId["visit"], 1) 

112 self.assertEqual(task.getName(), "test") 

113 config = dataRef.get("test_config", immediate=True) 

114 self.assertEqual(config, task.config) 

115 metadata = dataRef.get("test_metadata", immediate=True) 

116 self.assertEqual(metadata.getScalar("test.numProcessed"), 1) 

117 

118 def testOverrides(self): 

119 """Test config and log override 

120 """ 

121 config = ExampleTask.ConfigClass() 

122 config.floatField = -99.9 

123 log = Log.getLogger("cmdLineTask") 

124 retVal = ExampleTask.parseAndRun( 

125 args=[DataPath, "--output", self.outPath, "--id", "visit=2"], 

126 config=config, 

127 log=log 

128 ) 

129 self.assertEqual(retVal.parsedCmd.config.floatField, -99.9) 

130 self.assertIs(retVal.parsedCmd.log, log) 

131 

132 def testDoReturnResults(self): 

133 """Test the doReturnResults flag 

134 """ 

135 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

136 "--id", "visit=3", "filter=r"], doReturnResults=True) 

137 self.assertEqual(len(retVal.resultList), 1) 

138 result = retVal.resultList[0] 

139 self.assertEqual(result.metadata.getScalar("numProcessed"), 1) 

140 self.assertEqual(result.result.numProcessed, 1) 

141 

142 def testDoReturnResultsOnFailure(self): 

143 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

144 "--id", "visit=3", "filter=r", "--config", "doFail=True", 

145 "--clobber-config", "--noExit"], doReturnResults=True) 

146 self.assertEqual(len(retVal.resultList), 1) 

147 result = retVal.resultList[0] 

148 self.assertEqual(result.metadata.getScalar("numProcessed"), 0) 

149 self.assertEqual(retVal.resultList[0].result, None) 

150 

151 def testBackupConfig(self): 

152 """Test backup config file creation 

153 """ 

154 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"]) 

155 # Rerun with --clobber-config to ensure backup config file is created 

156 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r", 

157 "--config", "floatField=-99.9", "--clobber-config"]) 

158 # Ensure backup config file was created 

159 self.assertTrue(os.path.exists(os.path.join( 

160 self.outPath, "config", ExampleTask._DefaultName + ".py~1"))) 

161 

162 def testNoBackupConfig(self): 

163 """Test no backup config file creation 

164 """ 

165 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"]) 

166 # Rerun with --clobber-config and --no-backup-config to ensure backup config file is NOT created 

167 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r", 

168 "--config", "floatField=-99.9", "--clobber-config", 

169 "--no-backup-config"]) 

170 # Ensure backup config file was NOT created 

171 self.assertFalse( 

172 os.path.exists(os.path.join(self.outPath, "config", ExampleTask._DefaultName + ".py~1"))) 

173 

174 def testMultiprocess(self): 

175 """Test multiprocessing at a very minimal level 

176 """ 

177 for TaskClass in (ExampleTask, NoMultiprocessTask): 

178 result = TaskClass.parseAndRun(args=[DataPath, "--output", self.outPath, 

179 "-j", "5", "--id", "visit=2", "filter=r"]) 

180 self.assertEqual(result.taskRunner.numProcesses, 5 if TaskClass.canMultiprocess else 1) 

181 

182 def testCannotConstructTask(self): 

183 """Test error handling when a task cannot be constructed 

184 """ 

185 for doRaise in (False, True): 

186 args = [DataPath, "--output", self.outPath, "--id", "visit=1"] 

187 if doRaise: 

188 args.append("--doraise") 

189 with self.assertRaises(RuntimeError): 

190 CannotConstructTask.parseAndRun(args=args) 

191 

192 def testLegacyTask(self): 

193 """Test error handling when a task cannot be constructed 

194 """ 

195 retVal = LegacyTask.parseAndRun(args=[DataPath, "--output", self.outPath, 

196 "--id", "visit=3", "filter=r"], doReturnResults=True) 

197 self.assertEqual(retVal.resultList[0].result.didEnterRun, True) 

198 

199 

200class EaxmpleMultipleIdTaskRunner(pipeBase.TaskRunner): 

201 """TaskRunner to get multiple identifiers down into a Task""" 

202 @staticmethod 

203 def getTargetList(parsedCmd): 

204 """We want our Task to process one dataRef from each identifier at a time""" 

205 return list(zip(parsedCmd.one.refList, parsedCmd.two.refList)) 

206 

207 def __call__(self, target): 

208 """Send results from the Task back so we can inspect 

209 

210 For this test case with obs_test, we know that the results are picklable 

211 and small, so returning something is not a problem. 

212 """ 

213 task = self.TaskClass(config=self.config, log=self.log) 

214 return task.runDataRef(target) 

215 

216 

217class ExampleMultipleIdTask(pipeBase.CmdLineTask): 

218 _DefaultName = "test" 

219 ConfigClass = lsst.obs.test.TestConfig 

220 RunnerClass = EaxmpleMultipleIdTaskRunner 

221 

222 @classmethod 

223 def _makeArgumentParser(cls): 

224 """We want an argument parser that has multiple identifiers""" 

225 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

226 parser.add_id_argument("--one", "raw", "data identifier one", level="sensor") 

227 parser.add_id_argument("--two", "raw", "data identifier two", level="sensor") 

228 return parser 

229 

230 def runDataRef(self, data): 

231 """Our Task just spits back what's in the dataRefs.""" 

232 oneRef = data[0] 

233 twoRef = data[1] 

234 return oneRef.get("raw", snap=0, channel="0,0"), twoRef.get("raw", snap=0, channel="0,0") 

235 

236 

237class MultipleIdTaskTestCase(unittest.TestCase): 

238 """A test case for CmdLineTask using multiple identifiers 

239 

240 Tests implementation of ticket 2144, and demonstrates how 

241 to get results from multiple identifiers down into a Task. 

242 """ 

243 

244 def setUp(self): 

245 os.environ.pop("PIPE_INPUT_ROOT", None) 

246 os.environ.pop("PIPE_CALIB_ROOT", None) 

247 os.environ.pop("PIPE_OUTPUT_ROOT", None) 

248 self.outPath = tempfile.mkdtemp() 

249 

250 def tearDown(self): 

251 try: 

252 shutil.rmtree(self.outPath) 

253 except Exception: 

254 print("WARNING: failed to remove temporary dir %r" % (self.outPath,)) 

255 del self.outPath 

256 

257 def testMultiple(self): 

258 """Test use of a CmdLineTask with multiple identifiers""" 

259 args = [DataPath, "--output", self.outPath, 

260 "--one", "visit=1", "filter=g", 

261 "--two", "visit=2", "filter=g", 

262 ] 

263 retVal = ExampleMultipleIdTask.parseAndRun(args=args) 

264 self.assertEqual(len(retVal.resultList), 1) 

265 

266 

267class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

268 pass 

269 

270 

271def setup_module(module): 

272 lsst.utils.tests.init() 

273 

274 

275if __name__ == "__main__": 275 ↛ 276line 275 didn't jump to line 276, because the condition on line 275 was never true

276 lsst.utils.tests.init() 

277 unittest.main()