Coverage for tests/test_cmdLineTask.py: 35%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2015 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import shutil
24import unittest
25import tempfile
27import lsst.utils
28import lsst.pipe.base as pipeBase
29import lsst.obs.test
30from lsst.utils.timer import timeMethod
31import logging
33ObsTestDir = lsst.utils.getPackageDir("obs_test")
34DataPath = os.path.join(ObsTestDir, "data", "input")
37class ExampleTask(pipeBase.CmdLineTask):
38 ConfigClass = lsst.obs.test.TestConfig
39 _DefaultName = "test"
41 def __init__(self, *args, **kwargs):
42 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
43 self.dataRefList = []
44 self.numProcessed = 0
45 self.metadata.set("numProcessed", self.numProcessed)
47 @timeMethod
48 def runDataRef(self, dataRef):
49 if self.config.doFail:
50 raise pipeBase.TaskError("Failed by request: config.doFail is true")
51 self.dataRefList.append(dataRef)
52 self.numProcessed += 1
53 self.metadata.set("numProcessed", self.numProcessed)
54 return pipeBase.Struct(
55 numProcessed=self.numProcessed,
56 )
59class CannotConstructTask(ExampleTask):
60 """A task that cannot be constructed; used to test error handling
61 """
63 def __init__(self, *args, **kwargs):
64 raise RuntimeError("This task cannot be constructed")
67class NoMultiprocessTask(ExampleTask):
68 """Version of ExampleTask that does not support multiprocessing"""
69 canMultiprocess = False
72class LegacyTask(ExampleTask):
73 """Version of ExampleTask with `run` as entry point rather than
74 `runDataRef`
75 """
76 RunnerClass = pipeBase.LegacyTaskRunner
78 def run(self, dataRef):
79 results = self.runDataRef(dataRef)
80 resultsToBeAdded = pipeBase.Struct(didEnterRun=True)
81 results.mergeItems(resultsToBeAdded, "didEnterRun")
82 return results
85class CmdLineTaskTestCase(unittest.TestCase):
86 """A test case for CmdLineTask
87 """
89 def setUp(self):
90 os.environ.pop("PIPE_INPUT_ROOT", None)
91 os.environ.pop("PIPE_CALIB_ROOT", None)
92 os.environ.pop("PIPE_OUTPUT_ROOT", None)
93 self.outPath = tempfile.mkdtemp()
95 def tearDown(self):
96 try:
97 shutil.rmtree(self.outPath)
98 except Exception:
99 print("WARNING: failed to remove temporary dir %r" % (self.outPath,))
100 del self.outPath
102 def testBasics(self):
103 """Test basic construction and use of a command-line task
104 """
105 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=1"])
106 self.assertEqual(retVal.resultList, [pipeBase.Struct(exitStatus=0)])
107 task = ExampleTask(config=retVal.parsedCmd.config)
108 parsedCmd = retVal.parsedCmd
109 self.assertEqual(len(parsedCmd.id.refList), 1)
110 dataRef = parsedCmd.id.refList[0]
111 dataId = dataRef.dataId
112 self.assertEqual(dataId["visit"], 1)
113 self.assertEqual(task.getName(), "test")
114 config = dataRef.get("test_config", immediate=True)
115 self.assertEqual(config, task.config)
116 metadata = dataRef.get("test_metadata", immediate=True)
117 self.assertEqual(metadata.getScalar("test.numProcessed"), 1)
119 def testOverrides(self):
120 """Test config and log override
121 """
122 config = ExampleTask.ConfigClass()
123 config.floatField = -99.9
124 log = logging.getLogger("cmdLineTask")
125 retVal = ExampleTask.parseAndRun(
126 args=[DataPath, "--output", self.outPath, "--id", "visit=2"],
127 config=config,
128 log=log
129 )
130 self.assertEqual(retVal.parsedCmd.config.floatField, -99.9)
132 # The logger class may have been changed but the logger name
133 # should still match.
134 self.assertEqual(retVal.parsedCmd.log.name, log.name)
136 def testDoReturnResults(self):
137 """Test the doReturnResults flag
138 """
139 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath,
140 "--id", "visit=3", "filter=r"], doReturnResults=True)
141 self.assertEqual(len(retVal.resultList), 1)
142 result = retVal.resultList[0]
143 self.assertEqual(result.metadata.getScalar("numProcessed"), 1)
144 self.assertEqual(result.result.numProcessed, 1)
146 def testDoReturnResultsOnFailure(self):
147 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath,
148 "--id", "visit=3", "filter=r", "--config", "doFail=True",
149 "--clobber-config", "--noExit"], doReturnResults=True)
150 self.assertEqual(len(retVal.resultList), 1)
151 result = retVal.resultList[0]
152 self.assertEqual(result.metadata.getScalar("numProcessed"), 0)
153 self.assertEqual(retVal.resultList[0].result, None)
155 def testBackupConfig(self):
156 """Test backup config file creation
157 """
158 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"])
159 # Rerun with --clobber-config to ensure backup config file is created
160 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r",
161 "--config", "floatField=-99.9", "--clobber-config"])
162 # Ensure backup config file was created
163 self.assertTrue(os.path.exists(os.path.join(
164 self.outPath, "config", ExampleTask._DefaultName + ".py~1")))
166 def testNoBackupConfig(self):
167 """Test no backup config file creation
168 """
169 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"])
170 # Rerun with --clobber-config and --no-backup-config to ensure backup config file is NOT created
171 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r",
172 "--config", "floatField=-99.9", "--clobber-config",
173 "--no-backup-config"])
174 # Ensure backup config file was NOT created
175 self.assertFalse(
176 os.path.exists(os.path.join(self.outPath, "config", ExampleTask._DefaultName + ".py~1")))
178 def testMultiprocess(self):
179 """Test multiprocessing at a very minimal level
180 """
181 for TaskClass in (ExampleTask, NoMultiprocessTask):
182 result = TaskClass.parseAndRun(args=[DataPath, "--output", self.outPath,
183 "-j", "5", "--id", "visit=2", "filter=r"])
184 self.assertEqual(result.taskRunner.numProcesses, 5 if TaskClass.canMultiprocess else 1)
186 def testCannotConstructTask(self):
187 """Test error handling when a task cannot be constructed
188 """
189 for doRaise in (False, True):
190 args = [DataPath, "--output", self.outPath, "--id", "visit=1"]
191 if doRaise:
192 args.append("--doraise")
193 with self.assertRaises(RuntimeError):
194 CannotConstructTask.parseAndRun(args=args)
196 def testLegacyTask(self):
197 """Test error handling when a task cannot be constructed
198 """
199 retVal = LegacyTask.parseAndRun(args=[DataPath, "--output", self.outPath,
200 "--id", "visit=3", "filter=r"], doReturnResults=True)
201 self.assertEqual(retVal.resultList[0].result.didEnterRun, True)
204class EaxmpleMultipleIdTaskRunner(pipeBase.TaskRunner):
205 """TaskRunner to get multiple identifiers down into a Task"""
206 @staticmethod
207 def getTargetList(parsedCmd):
208 """We want our Task to process one dataRef from each identifier at a time"""
209 return list(zip(parsedCmd.one.refList, parsedCmd.two.refList))
211 def __call__(self, target):
212 """Send results from the Task back so we can inspect
214 For this test case with obs_test, we know that the results are picklable
215 and small, so returning something is not a problem.
216 """
217 task = self.TaskClass(config=self.config, log=self.log)
218 return task.runDataRef(target)
221class ExampleMultipleIdTask(pipeBase.CmdLineTask):
222 _DefaultName = "test"
223 ConfigClass = lsst.obs.test.TestConfig
224 RunnerClass = EaxmpleMultipleIdTaskRunner
226 @classmethod
227 def _makeArgumentParser(cls):
228 """We want an argument parser that has multiple identifiers"""
229 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
230 parser.add_id_argument("--one", "raw", "data identifier one", level="sensor")
231 parser.add_id_argument("--two", "raw", "data identifier two", level="sensor")
232 return parser
234 def runDataRef(self, data):
235 """Our Task just spits back what's in the dataRefs."""
236 oneRef = data[0]
237 twoRef = data[1]
238 return oneRef.get("raw", snap=0, channel="0,0"), twoRef.get("raw", snap=0, channel="0,0")
241class MultipleIdTaskTestCase(unittest.TestCase):
242 """A test case for CmdLineTask using multiple identifiers
244 Tests implementation of ticket 2144, and demonstrates how
245 to get results from multiple identifiers down into a Task.
246 """
248 def setUp(self):
249 os.environ.pop("PIPE_INPUT_ROOT", None)
250 os.environ.pop("PIPE_CALIB_ROOT", None)
251 os.environ.pop("PIPE_OUTPUT_ROOT", None)
252 self.outPath = tempfile.mkdtemp()
254 def tearDown(self):
255 try:
256 shutil.rmtree(self.outPath)
257 except Exception:
258 print("WARNING: failed to remove temporary dir %r" % (self.outPath,))
259 del self.outPath
261 def testMultiple(self):
262 """Test use of a CmdLineTask with multiple identifiers"""
263 args = [DataPath, "--output", self.outPath,
264 "--one", "visit=1", "filter=g",
265 "--two", "visit=2", "filter=g",
266 ]
267 retVal = ExampleMultipleIdTask.parseAndRun(args=args)
268 self.assertEqual(len(retVal.resultList), 1)
271class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
272 pass
275def setup_module(module):
276 lsst.utils.tests.init()
279if __name__ == "__main__": 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true
280 lsst.utils.tests.init()
281 unittest.main()