Coverage for tests/test_cmdLineTask.py: 35%
149 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:08 -0800
« prev ^ index » next coverage.py v7.1.0, created at 2023-02-05 18:08 -0800
1#
2# LSST Data Management System
3# Copyright 2008-2015 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import shutil
24import unittest
25import tempfile
27import lsst.utils
28import lsst.pipe.base as pipeBase
29import lsst.obs.test
30import logging
32ObsTestDir = lsst.utils.getPackageDir("obs_test")
33DataPath = os.path.join(ObsTestDir, "data", "input")
36class ExampleTask(pipeBase.CmdLineTask):
37 ConfigClass = lsst.obs.test.TestConfig
38 _DefaultName = "test"
40 def __init__(self, *args, **kwargs):
41 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
42 self.dataRefList = []
43 self.numProcessed = 0
44 self.metadata.set("numProcessed", self.numProcessed)
46 @pipeBase.timeMethod
47 def runDataRef(self, dataRef):
48 if self.config.doFail:
49 raise pipeBase.TaskError("Failed by request: config.doFail is true")
50 self.dataRefList.append(dataRef)
51 self.numProcessed += 1
52 self.metadata.set("numProcessed", self.numProcessed)
53 return pipeBase.Struct(
54 numProcessed=self.numProcessed,
55 )
58class CannotConstructTask(ExampleTask):
59 """A task that cannot be constructed; used to test error handling
60 """
62 def __init__(self, *args, **kwargs):
63 raise RuntimeError("This task cannot be constructed")
66class NoMultiprocessTask(ExampleTask):
67 """Version of ExampleTask that does not support multiprocessing"""
68 canMultiprocess = False
71class LegacyTask(ExampleTask):
72 """Version of ExampleTask with `run` as entry point rather than
73 `runDataRef`
74 """
75 RunnerClass = pipeBase.LegacyTaskRunner
77 def run(self, dataRef):
78 results = self.runDataRef(dataRef)
79 resultsToBeAdded = pipeBase.Struct(didEnterRun=True)
80 results.mergeItems(resultsToBeAdded, "didEnterRun")
81 return results
84class CmdLineTaskTestCase(unittest.TestCase):
85 """A test case for CmdLineTask
86 """
88 def setUp(self):
89 os.environ.pop("PIPE_INPUT_ROOT", None)
90 os.environ.pop("PIPE_CALIB_ROOT", None)
91 os.environ.pop("PIPE_OUTPUT_ROOT", None)
92 self.outPath = tempfile.mkdtemp()
94 def tearDown(self):
95 try:
96 shutil.rmtree(self.outPath)
97 except Exception:
98 print("WARNING: failed to remove temporary dir %r" % (self.outPath,))
99 del self.outPath
101 def testBasics(self):
102 """Test basic construction and use of a command-line task
103 """
104 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=1"])
105 self.assertEqual(retVal.resultList, [pipeBase.Struct(exitStatus=0)])
106 task = ExampleTask(config=retVal.parsedCmd.config)
107 parsedCmd = retVal.parsedCmd
108 self.assertEqual(len(parsedCmd.id.refList), 1)
109 dataRef = parsedCmd.id.refList[0]
110 dataId = dataRef.dataId
111 self.assertEqual(dataId["visit"], 1)
112 self.assertEqual(task.getName(), "test")
113 config = dataRef.get("test_config", immediate=True)
114 self.assertEqual(config, task.config)
115 metadata = dataRef.get("test_metadata", immediate=True)
116 self.assertEqual(metadata.getScalar("test.numProcessed"), 1)
118 def testOverrides(self):
119 """Test config and log override
120 """
121 config = ExampleTask.ConfigClass()
122 config.floatField = -99.9
123 log = logging.getLogger("cmdLineTask")
124 retVal = ExampleTask.parseAndRun(
125 args=[DataPath, "--output", self.outPath, "--id", "visit=2"],
126 config=config,
127 log=log
128 )
129 self.assertEqual(retVal.parsedCmd.config.floatField, -99.9)
131 # The logger class may have been changed but the logger name
132 # should still match.
133 self.assertEqual(retVal.parsedCmd.log.name, log.name)
135 def testDoReturnResults(self):
136 """Test the doReturnResults flag
137 """
138 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath,
139 "--id", "visit=3", "filter=r"], doReturnResults=True)
140 self.assertEqual(len(retVal.resultList), 1)
141 result = retVal.resultList[0]
142 self.assertEqual(result.metadata.getScalar("numProcessed"), 1)
143 self.assertEqual(result.result.numProcessed, 1)
145 def testDoReturnResultsOnFailure(self):
146 retVal = ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath,
147 "--id", "visit=3", "filter=r", "--config", "doFail=True",
148 "--clobber-config", "--noExit"], doReturnResults=True)
149 self.assertEqual(len(retVal.resultList), 1)
150 result = retVal.resultList[0]
151 self.assertEqual(result.metadata.getScalar("numProcessed"), 0)
152 self.assertEqual(retVal.resultList[0].result, None)
154 def testBackupConfig(self):
155 """Test backup config file creation
156 """
157 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"])
158 # Rerun with --clobber-config to ensure backup config file is created
159 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r",
160 "--config", "floatField=-99.9", "--clobber-config"])
161 # Ensure backup config file was created
162 self.assertTrue(os.path.exists(os.path.join(
163 self.outPath, "config", ExampleTask._DefaultName + ".py~1")))
165 def testNoBackupConfig(self):
166 """Test no backup config file creation
167 """
168 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r"])
169 # Rerun with --clobber-config and --no-backup-config to ensure backup config file is NOT created
170 ExampleTask.parseAndRun(args=[DataPath, "--output", self.outPath, "--id", "visit=3", "filter=r",
171 "--config", "floatField=-99.9", "--clobber-config",
172 "--no-backup-config"])
173 # Ensure backup config file was NOT created
174 self.assertFalse(
175 os.path.exists(os.path.join(self.outPath, "config", ExampleTask._DefaultName + ".py~1")))
177 def testMultiprocess(self):
178 """Test multiprocessing at a very minimal level
179 """
180 for TaskClass in (ExampleTask, NoMultiprocessTask):
181 result = TaskClass.parseAndRun(args=[DataPath, "--output", self.outPath,
182 "-j", "5", "--id", "visit=2", "filter=r"])
183 self.assertEqual(result.taskRunner.numProcesses, 5 if TaskClass.canMultiprocess else 1)
185 def testCannotConstructTask(self):
186 """Test error handling when a task cannot be constructed
187 """
188 for doRaise in (False, True):
189 args = [DataPath, "--output", self.outPath, "--id", "visit=1"]
190 if doRaise:
191 args.append("--doraise")
192 with self.assertRaises(RuntimeError):
193 CannotConstructTask.parseAndRun(args=args)
195 def testLegacyTask(self):
196 """Test error handling when a task cannot be constructed
197 """
198 retVal = LegacyTask.parseAndRun(args=[DataPath, "--output", self.outPath,
199 "--id", "visit=3", "filter=r"], doReturnResults=True)
200 self.assertEqual(retVal.resultList[0].result.didEnterRun, True)
203class EaxmpleMultipleIdTaskRunner(pipeBase.TaskRunner):
204 """TaskRunner to get multiple identifiers down into a Task"""
205 @staticmethod
206 def getTargetList(parsedCmd):
207 """We want our Task to process one dataRef from each identifier at a time"""
208 return list(zip(parsedCmd.one.refList, parsedCmd.two.refList))
210 def __call__(self, target):
211 """Send results from the Task back so we can inspect
213 For this test case with obs_test, we know that the results are picklable
214 and small, so returning something is not a problem.
215 """
216 task = self.TaskClass(config=self.config, log=self.log)
217 return task.runDataRef(target)
220class ExampleMultipleIdTask(pipeBase.CmdLineTask):
221 _DefaultName = "test"
222 ConfigClass = lsst.obs.test.TestConfig
223 RunnerClass = EaxmpleMultipleIdTaskRunner
225 @classmethod
226 def _makeArgumentParser(cls):
227 """We want an argument parser that has multiple identifiers"""
228 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
229 parser.add_id_argument("--one", "raw", "data identifier one", level="sensor")
230 parser.add_id_argument("--two", "raw", "data identifier two", level="sensor")
231 return parser
233 def runDataRef(self, data):
234 """Our Task just spits back what's in the dataRefs."""
235 oneRef = data[0]
236 twoRef = data[1]
237 return oneRef.get("raw", snap=0, channel="0,0"), twoRef.get("raw", snap=0, channel="0,0")
240class MultipleIdTaskTestCase(unittest.TestCase):
241 """A test case for CmdLineTask using multiple identifiers
243 Tests implementation of ticket 2144, and demonstrates how
244 to get results from multiple identifiers down into a Task.
245 """
247 def setUp(self):
248 os.environ.pop("PIPE_INPUT_ROOT", None)
249 os.environ.pop("PIPE_CALIB_ROOT", None)
250 os.environ.pop("PIPE_OUTPUT_ROOT", None)
251 self.outPath = tempfile.mkdtemp()
253 def tearDown(self):
254 try:
255 shutil.rmtree(self.outPath)
256 except Exception:
257 print("WARNING: failed to remove temporary dir %r" % (self.outPath,))
258 del self.outPath
260 def testMultiple(self):
261 """Test use of a CmdLineTask with multiple identifiers"""
262 args = [DataPath, "--output", self.outPath,
263 "--one", "visit=1", "filter=g",
264 "--two", "visit=2", "filter=g",
265 ]
266 retVal = ExampleMultipleIdTask.parseAndRun(args=args)
267 self.assertEqual(len(retVal.resultList), 1)
270class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
271 pass
274def setup_module(module):
275 lsst.utils.tests.init()
278if __name__ == "__main__": 278 ↛ 279line 278 didn't jump to line 279, because the condition on line 278 was never true
279 lsst.utils.tests.init()
280 unittest.main()