Coverage for tests/test_appipe.py : 26%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_pipe.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import contextlib
25import os
26import unittest
27from unittest.mock import patch, Mock, ANY
29import lsst.utils.tests
30import lsst.pex.exceptions as pexExcept
31import lsst.daf.persistence as dafPersist
32import lsst.pipe.base as pipeBase
34from lsst.ap.pipe import ApPipeTask
37class PipelineTestSuite(lsst.utils.tests.TestCase):
38 '''
39 A set of tests for the functions in ap_pipe.
40 '''
42 @classmethod
43 def _makeDefaultConfig(cls):
44 config = ApPipeTask.ConfigClass()
45 config.load(os.path.join(cls.datadir, "config", "apPipe.py"))
46 config.diaPipe.apdb.db_url = "sqlite://"
47 config.diaPipe.apdb.isolation_level = "READ_UNCOMMITTED"
48 return config
50 @classmethod
51 def setUpClass(cls):
52 try:
53 cls.datadir = lsst.utils.getPackageDir("ap_pipe_testdata")
54 except pexExcept.NotFoundError:
55 raise unittest.SkipTest("ap_pipe_testdata not set up")
57 def _setupObjPatch(self, *args, **kwargs):
58 """Create a patch in setUp that will be reverted once the test ends.
60 Parameters
61 ----------
62 *args
63 **kwargs
64 Inputs to `unittest.mock.patch.object`.
65 """
66 patcher = patch.object(*args, **kwargs)
67 patcher.start()
68 self.addCleanup(patcher.stop)
70 def setUp(self):
71 self.config = self._makeDefaultConfig()
72 self.butler = dafPersist.Butler(inputs={'root': self.datadir})
74 def makeMockDataRef(datasetType, level=None, dataId={}, **rest):
75 mockDataRef = Mock(dafPersist.ButlerDataRef)
76 mockDataRef.dataId = dict(dataId, **rest)
77 return mockDataRef
79 self._setupObjPatch(self.butler, "dataRef", side_effect=makeMockDataRef)
80 self.dataId = {"visit": 413635, "ccdnum": 42}
81 self.inputRef = self.butler.dataRef("raw", **self.dataId)
83 @contextlib.contextmanager
84 def mockPatchSubtasks(self, task):
85 """Make mocks for all the ap_pipe subtasks.
87 This is needed because the task itself cannot be a mock.
88 The task's subtasks do not exist until the task is created, so
89 this allows us to mock them instead.
91 Parameters
92 ----------
93 task : `lsst.ap.pipe.ApPipeTask`
94 The task whose subtasks will be mocked.
96 Yields
97 ------
98 subtasks : `lsst.pipe.base.Struct`
99 All mocks created by this context manager, including:
101 ``ccdProcessor``
102 ``differencer``
103 ``diaPipe``
104 a mock for the corresponding subtask. Mocks do not return any
105 particular value, but have mocked methods that can be queried
106 for calls by ApPipeTask
107 """
108 with patch.object(task, "ccdProcessor") as mockCcdProcessor, \
109 patch.object(task, "differencer") as mockDifferencer, \
110 patch.object(task, "diaPipe") as mockDiaPipe:
111 yield pipeBase.Struct(ccdProcessor=mockCcdProcessor,
112 differencer=mockDifferencer,
113 diaPipe=mockDiaPipe)
115 def testGenericRun(self):
116 """Test the normal workflow of each ap_pipe step.
117 """
118 task = ApPipeTask(self.butler, config=self.config)
119 with self.mockPatchSubtasks(task) as subtasks:
120 task.runDataRef(self.inputRef)
121 subtasks.ccdProcessor.runDataRef.assert_called_once()
122 subtasks.differencer.runDataRef.assert_called_once()
123 subtasks.diaPipe.run.assert_called_once()
125 def testReuseExistingOutput(self):
126 """Test reuse keyword to ApPipeTask.runDataRef.
127 """
128 task = ApPipeTask(self.butler, config=self.config)
130 self.checkReuseExistingOutput(task, ['ccdProcessor'])
131 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer'])
132 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer', 'diaPipe'])
134 def checkReuseExistingOutput(self, task, skippable):
135 """Check whether a task's subtasks are skipped when "reuse" is set.
137 Mock guarantees that all "has this been made" tests pass,
138 so skippable subtasks should actually be skipped.
139 """
140 # extremely brittle because it depends on internal code of runDataRef
141 # but it only needs to work until DM-21886, then we can unit-test the subtask
142 internalRef = self.inputRef.getButler().dataRef()
143 internalRef.put.reset_mock()
145 with self.mockPatchSubtasks(task) as subtasks:
146 struct = task.runDataRef(self.inputRef, reuse=skippable)
147 for subtaskName, runner in {
148 'ccdProcessor': subtasks.ccdProcessor.runDataRef,
149 'differencer': subtasks.differencer.runDataRef,
150 'diaPipe': subtasks.diaPipe.run,
151 }.items():
152 msg = "subtask = " + subtaskName
153 if subtaskName in skippable:
154 runner.assert_not_called()
155 self.assertIsNone(struct.getDict()[subtaskName], msg=msg)
156 else:
157 runner.assert_called_once()
158 self.assertIsNotNone(struct.getDict()[subtaskName], msg=msg)
160 if 'diaPipe' in skippable:
161 internalRef.put.assert_not_called()
162 else:
163 internalRef.put.assert_called_once_with(ANY, "apdb_marker")
165 def testCalexpRun(self):
166 """Test the calexp template workflow of each ap_pipe step.
167 """
168 calexpConfigFile = os.path.join(lsst.utils.getPackageDir('ap_pipe'),
169 'config', 'calexpTemplates.py')
170 calexpConfig = self._makeDefaultConfig()
171 calexpConfig.load(calexpConfigFile)
172 calexpConfig.differencer.doSelectSources = False # Workaround for DM-18394
174 task = ApPipeTask(self.butler, config=calexpConfig)
175 with self.mockPatchSubtasks(task) as subtasks:
176 # We use the same dataId here for both template and science
177 # in difference imaging. This is OK because everything is a mock
178 # and we aren't actually doing any image processing.
179 task.runDataRef(self.inputRef, templateIds=[self.dataId])
180 self.assertEqual(subtasks.ccdProcessor.runDataRef.call_count, 2)
181 subtasks.differencer.runDataRef.assert_called_once()
182 subtasks.diaPipe.run.assert_called_once()
185class MemoryTester(lsst.utils.tests.MemoryTestCase):
186 pass
189def setup_module(module):
190 lsst.utils.tests.init()
193if __name__ == "__main__": 193 ↛ 194line 193 didn't jump to line 194, because the condition on line 193 was never true
194 lsst.utils.tests.init()
195 unittest.main()