Coverage for tests/test_appipe.py : 28%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_pipe.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import contextlib
25import os
26import unittest
27from unittest.mock import patch, Mock, ANY
29import lsst.utils.tests
30import lsst.pex.exceptions as pexExcept
31import lsst.daf.persistence as dafPersist
32import lsst.pipe.base as pipeBase
34from lsst.ap.pipe import ApPipeTask
37class PipelineTestSuite(lsst.utils.tests.TestCase):
38 '''
39 A set of tests for the functions in ap_pipe.
40 '''
42 @classmethod
43 def _makeDefaultConfig(cls):
44 config = ApPipeTask.ConfigClass()
45 config.load(os.path.join(cls.datadir, "config", "apPipe.py"))
46 config.diaPipe.apdb.db_url = "sqlite://"
47 config.diaPipe.apdb.isolation_level = "READ_UNCOMMITTED"
48 return config
50 @classmethod
51 def setUpClass(cls):
52 try:
53 cls.datadir = lsst.utils.getPackageDir("ap_pipe_testdata")
54 except pexExcept.NotFoundError:
55 raise unittest.SkipTest("ap_pipe_testdata not set up")
56 try:
57 lsst.utils.getPackageDir("obs_decam")
58 except LookupError:
59 raise unittest.SkipTest("obs_decam not set up; needed for ap_pipe_testdata")
61 def _setupObjPatch(self, *args, **kwargs):
62 """Create a patch in setUp that will be reverted once the test ends.
64 Parameters
65 ----------
66 *args
67 **kwargs
68 Inputs to `unittest.mock.patch.object`.
69 """
70 patcher = patch.object(*args, **kwargs)
71 patcher.start()
72 self.addCleanup(patcher.stop)
74 def setUp(self):
75 self.config = self._makeDefaultConfig()
76 self.butler = dafPersist.Butler(inputs={'root': self.datadir})
78 def makeMockDataRef(datasetType, level=None, dataId={}, **rest):
79 mockDataRef = Mock(dafPersist.ButlerDataRef)
80 mockDataRef.dataId = dict(dataId, **rest)
81 return mockDataRef
83 self._setupObjPatch(self.butler, "dataRef", side_effect=makeMockDataRef)
84 self.dataId = {"visit": 413635, "ccdnum": 42}
85 self.inputRef = self.butler.dataRef("raw", **self.dataId)
87 @contextlib.contextmanager
88 def mockPatchSubtasks(self, task):
89 """Make mocks for all the ap_pipe subtasks.
91 This is needed because the task itself cannot be a mock.
92 The task's subtasks do not exist until the task is created, so
93 this allows us to mock them instead.
95 Parameters
96 ----------
97 task : `lsst.ap.pipe.ApPipeTask`
98 The task whose subtasks will be mocked.
100 Yields
101 ------
102 subtasks : `lsst.pipe.base.Struct`
103 All mocks created by this context manager, including:
105 ``ccdProcessor``
106 ``differencer``
107 ``diaPipe``
108 a mock for the corresponding subtask. Mocks do not return any
109 particular value, but have mocked methods that can be queried
110 for calls by ApPipeTask
111 """
112 with patch.object(task, "ccdProcessor") as mockCcdProcessor, \
113 patch.object(task, "differencer") as mockDifferencer, \
114 patch.object(task, "transformDiaSrcCat") as mockTransform, \
115 patch.object(task, "diaPipe") as mockDiaPipe:
116 yield pipeBase.Struct(ccdProcessor=mockCcdProcessor,
117 differencer=mockDifferencer,
118 transformDiaSrcCat=mockTransform,
119 diaPipe=mockDiaPipe)
121 def testGenericRun(self):
122 """Test the normal workflow of each ap_pipe step.
123 """
124 task = ApPipeTask(self.butler, config=self.config)
125 with self.mockPatchSubtasks(task) as subtasks:
126 task.runDataRef(self.inputRef)
127 subtasks.ccdProcessor.runDataRef.assert_called_once()
128 subtasks.differencer.runDataRef.assert_called_once()
129 subtasks.diaPipe.run.assert_called_once()
131 def testReuseExistingOutput(self):
132 """Test reuse keyword to ApPipeTask.runDataRef.
133 """
134 task = ApPipeTask(self.butler, config=self.config)
136 self.checkReuseExistingOutput(task, ['ccdProcessor'])
137 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer'])
138 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer', 'diaPipe'])
140 def checkReuseExistingOutput(self, task, skippable):
141 """Check whether a task's subtasks are skipped when "reuse" is set.
143 Mock guarantees that all "has this been made" tests pass,
144 so skippable subtasks should actually be skipped.
145 """
146 # extremely brittle because it depends on internal code of runDataRef
147 # but it only needs to work until DM-21886, then we can unit-test the subtask
148 internalRef = self.inputRef.getButler().dataRef()
149 internalRef.put.reset_mock()
151 with self.mockPatchSubtasks(task) as subtasks:
152 struct = task.runDataRef(self.inputRef, reuse=skippable)
153 for subtaskName, runner in {
154 'ccdProcessor': subtasks.ccdProcessor.runDataRef,
155 'differencer': subtasks.differencer.runDataRef,
156 'diaPipe': subtasks.diaPipe.run,
157 }.items():
158 msg = "subtask = " + subtaskName
159 if subtaskName in skippable:
160 runner.assert_not_called()
161 self.assertIsNone(struct.getDict()[subtaskName], msg=msg)
162 else:
163 runner.assert_called_once()
164 self.assertIsNotNone(struct.getDict()[subtaskName], msg=msg)
166 if 'diaPipe' in skippable:
167 internalRef.put.assert_not_called()
168 else:
169 internalRef.put.assert_called_once_with(ANY, "apdb_marker")
171 def testCalexpRun(self):
172 """Test the calexp template workflow of each ap_pipe step.
173 """
174 calexpConfigFile = os.path.join(lsst.utils.getPackageDir('ap_pipe'),
175 'config', 'calexpTemplates.py')
176 calexpConfig = self._makeDefaultConfig()
177 calexpConfig.load(calexpConfigFile)
178 calexpConfig.differencer.doSelectSources = False # Workaround for DM-18394
180 task = ApPipeTask(self.butler, config=calexpConfig)
181 with self.mockPatchSubtasks(task) as subtasks:
182 # We use the same dataId here for both template and science
183 # in difference imaging. This is OK because everything is a mock
184 # and we aren't actually doing any image processing.
185 task.runDataRef(self.inputRef, templateIds=[self.dataId])
186 self.assertEqual(subtasks.ccdProcessor.runDataRef.call_count, 2)
187 subtasks.differencer.runDataRef.assert_called_once()
188 subtasks.diaPipe.run.assert_called_once()
191class MemoryTester(lsst.utils.tests.MemoryTestCase):
192 pass
195def setup_module(module):
196 lsst.utils.tests.init()
199if __name__ == "__main__": 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true
200 lsst.utils.tests.init()
201 unittest.main()