Coverage for tests/test_cmdLineTask.py : 33%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2015 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import shutil
24import unittest
25import tempfile
27import lsst.utils
28import lsst.pipe.base as pipeBase
29import lsst.obs.test
30from lsst.log import Log
32ObsTestDir = lsst.utils.getPackageDir("obs_test")
33DataPath = os.path.join(ObsTestDir, "data", "input")
36class ExampleTask(pipeBase.CmdLineTask):
37 ConfigClass = lsst.obs.test.TestConfig
38 _DefaultName = "test"
40 def __init__(self, *args, **kwargs):
41 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
42 self.dataRefList = []
43 self.numProcessed = 0
44 self.metadata.set("numProcessed", self.numProcessed)
46 @pipeBase.timeMethod
47 def runDataRef(self, dataRef):
48 if self.config.doFail:
49 raise pipeBase.TaskError("Failed by request: config.doFail is true")
50 self.dataRefList.append(dataRef)
51 self.numProcessed += 1
52 self.metadata.set("numProcessed", self.numProcessed)
53 return pipeBase.Struct(
54 numProcessed=self.numProcessed,
55 )
58class CannotConstructTask(ExampleTask):
59 """A task that cannot be constructed; used to test error handling
60 """
62 def __init__(self, *args, **kwargs):
63 raise RuntimeError("This task cannot be constructed")
66class NoMultiprocessTask(ExampleTask):
67 """Version of ExampleTask that does not support multiprocessing"""
68 canMultiprocess = False
71class LegacyTask(ExampleTask):
72 """Version of ExampleTask with `run` as entry point rather than
73 `runDataRef`
74 """
75 RunnerClass = pipeBase.LegacyTaskRunner
77 def run(self, dataRef):
78 results = self.runDataRef(dataRef)
79 resultsToBeAdded = pipeBase.Struct(didEnterRun=True)
80 results.mergeItems(resultsToBeAdded, "didEnterRun")
81 return results
84class CmdLineTaskTestCase(unittest.TestCase):
85 """A test case for CmdLineTask
86 """
88 def setUp(self):
89 os.environ.pop("PIPE_INPUT_ROOT", None)
90 os.environ.pop("PIPE_CALIB_ROOT", None)
91 os.environ.pop("PIPE_OUTPUT_ROOT", None)
92 self.outPath = tempfile.mkdtemp()
94 def tearDown(self):
95 try:
96 shutil.rmtree(self.outPath)
97 except Exception:
98 print("WARNING: failed to remove temporary dir %r" % (self.outPath,))
99 del self.outPath
101 def testBasics(self):
102 """Test basic construction and use of a command-line task
103 """
104 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=1"])
105 self.assertEqual(retVal.resultList, [pipeBase.Struct(exitStatus=0)])
106 task = ExampleTask(config=retVal.parsedCmd.config)
107 parsedCmd = retVal.parsedCmd
108 self.assertEqual(len(parsedCmd.id.refList), 1)
109 dataRef = parsedCmd.id.refList[0]
110 dataId = dataRef.dataId
111 self.assertEqual(dataId["visit"], 1)
112 self.assertEqual(task.getName(), "test")
113 config = dataRef.get("test_config", immediate=True)
114 self.assertEqual(config, task.config)
115 metadata = dataRef.get("test_metadata", immediate=True)
116 self.assertEqual(metadata.getScalar("test.numProcessed"), 1)
118 def testOverrides(self):
119 """Test config and log override
120 """
121 config = ExampleTask.ConfigClass()
122 config.floatField = -99.9
123 log = Log.getLogger("cmdLineTask")
124 retVal = ExampleTask.parseAndRun(
125 args=[DataPath, "--output", self.outPath, "--id", "visit=2"],
126 config=config,
127 log=log
128 )
129 self.assertEqual(retVal.parsedCmd.config.floatField, -99.9)
130 self.assertIs(retVal.parsedCmd.log, log)
132 def testDoReturnResults(self):
133 """Test the doReturnResults flag
134 """
135 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath,
136 "--id", "visit=3", "filter=r"], doReturnResults=True)
137 self.assertEqual(len(retVal.resultList), 1)
138 result = retVal.resultList[0]
139 self.assertEqual(result.metadata.getScalar("numProcessed"), 1)
140 self.assertEqual(result.result.numProcessed, 1)
142 def testDoReturnResultsOnFailure(self):
143 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath,
144 "--id", "visit=3", "filter=r", "--config", "doFail=True",
145 "--clobber-config", "--noExit"], doReturnResults=True)
146 self.assertEqual(len(retVal.resultList), 1)
147 result = retVal.resultList[0]
148 self.assertEqual(result.metadata.getScalar("numProcessed"), 0)
149 self.assertEqual(retVal.resultList[0].result, None)
151 def testBackupConfig(self):
152 """Test backup config file creation
153 """
154 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"])
155 # Rerun with --clobber-config to ensure backup config file is created
156 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r",
157 "--config", "floatField=-99.9", "--clobber-config"])
158 # Ensure backup config file was created
159 self.assertTrue(os.path.exists(os.path.join(
160 self.outPath, "config", ExampleTask._DefaultName + ".py~1")))
162 def testNoBackupConfig(self):
163 """Test no backup config file creation
164 """
165 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"])
166 # Rerun with --clobber-config and --no-backup-config to ensure backup config file is NOT created
167 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r",
168 "--config", "floatField=-99.9", "--clobber-config",
169 "--no-backup-config"])
170 # Ensure backup config file was NOT created
171 self.assertFalse(
172 os.path.exists(os.path.join(self.outPath, "config", ExampleTask._DefaultName + ".py~1")))
174 def testMultiprocess(self):
175 """Test multiprocessing at a very minimal level
176 """
177 for TaskClass in (ExampleTask, NoMultiprocessTask):
178 result = TaskClass.parseAndRun(args=[DataPath, "--output", self.outPath,
179 "-j", "5", "--id", "visit=2", "filter=r"])
180 self.assertEqual(result.taskRunner.numProcesses, 5 if TaskClass.canMultiprocess else 1)
182 def testCannotConstructTask(self):
183 """Test error handling when a task cannot be constructed
184 """
185 for doRaise in (False, True):
186 args = [DataPath, "--output", self.outPath, "--id", "visit=1"]
187 if doRaise:
188 args.append("--doraise")
189 with self.assertRaises(RuntimeError):
190 CannotConstructTask.parseAndRun(args=args)
192 def testLegacyTask(self):
193 """Test error handling when a task cannot be constructed
194 """
195 retVal = LegacyTask.parseAndRun(args=[DataPath, "--output", self.outPath,
196 "--id", "visit=3", "filter=r"], doReturnResults=True)
197 self.assertEqual(retVal.resultList[0].result.didEnterRun, True)
200class EaxmpleMultipleIdTaskRunner(pipeBase.TaskRunner):
201 """TaskRunner to get multiple identifiers down into a Task"""
202 @staticmethod
203 def getTargetList(parsedCmd):
204 """We want our Task to process one dataRef from each identifier at a time"""
205 return list(zip(parsedCmd.one.refList, parsedCmd.two.refList))
207 def __call__(self, target):
208 """Send results from the Task back so we can inspect
210 For this test case with obs_test, we know that the results are picklable
211 and small, so returning something is not a problem.
212 """
213 task = self.TaskClass(config=self.config, log=self.log)
214 return task.runDataRef(target)
217class ExampleMultipleIdTask(pipeBase.CmdLineTask):
218 _DefaultName = "test"
219 ConfigClass = lsst.obs.test.TestConfig
220 RunnerClass = EaxmpleMultipleIdTaskRunner
222 @classmethod
223 def _makeArgumentParser(cls):
224 """We want an argument parser that has multiple identifiers"""
225 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
226 parser.add_id_argument("--one", "raw", "data identifier one", level="sensor")
227 parser.add_id_argument("--two", "raw", "data identifier two", level="sensor")
228 return parser
230 def runDataRef(self, data):
231 """Our Task just spits back what's in the dataRefs."""
232 oneRef = data[0]
233 twoRef = data[1]
234 return oneRef.get("raw", snap=0, channel="0,0"), twoRef.get("raw", snap=0, channel="0,0")
237class MultipleIdTaskTestCase(unittest.TestCase):
238 """A test case for CmdLineTask using multiple identifiers
240 Tests implementation of ticket 2144, and demonstrates how
241 to get results from multiple identifiers down into a Task.
242 """
244 def setUp(self):
245 os.environ.pop("PIPE_INPUT_ROOT", None)
246 os.environ.pop("PIPE_CALIB_ROOT", None)
247 os.environ.pop("PIPE_OUTPUT_ROOT", None)
248 self.outPath = tempfile.mkdtemp()
250 def tearDown(self):
251 try:
252 shutil.rmtree(self.outPath)
253 except Exception:
254 print("WARNING: failed to remove temporary dir %r" % (self.outPath,))
255 del self.outPath
257 def testMultiple(self):
258 """Test use of a CmdLineTask with multiple identifiers"""
259 args = [DataPath, "--output", self.outPath,
260 "--one", "visit=1", "filter=g",
261 "--two", "visit=2", "filter=g",
262 ]
263 retVal = ExampleMultipleIdTask.parseAndRun(args=args)
264 self.assertEqual(len(retVal.resultList), 1)
267class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
268 pass
271def setup_module(module):
272 lsst.utils.tests.init()
275if __name__ == "__main__": 275 ↛ 276line 275 didn't jump to line 276, because the condition on line 275 was never true
276 lsst.utils.tests.init()
277 unittest.main()