Coverage for tests/test_appipe.py: 27%

89 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 21:33 +0000

1# 

2# This file is part of ap_pipe. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import contextlib 

25import os 

26import unittest 

27from unittest.mock import patch, Mock, ANY 

28 

29import lsst.utils.tests 

30import lsst.daf.persistence as dafPersist 

31import lsst.pipe.base as pipeBase 

32 

33from lsst.ap.pipe import ApPipeTask 

34 

35 

36class PipelineTestSuite(lsst.utils.tests.TestCase): 

37 ''' 

38 A set of tests for the functions in ap_pipe. 

39 ''' 

40 

41 @classmethod 

42 def _makeDefaultConfig(cls): 

43 config = ApPipeTask.ConfigClass() 

44 config.load(os.path.join(cls.datadir, "config", "apPipe.py")) 

45 config.diaPipe.apdb.db_url = "sqlite://" 

46 config.diaPipe.apdb.isolation_level = "READ_UNCOMMITTED" 

47 return config 

48 

49 @classmethod 

50 def setUpClass(cls): 

51 try: 

52 cls.datadir = lsst.utils.getPackageDir("ap_pipe_testdata") 

53 except LookupError: 

54 raise unittest.SkipTest("ap_pipe_testdata not set up") 

55 try: 

56 lsst.utils.getPackageDir("obs_decam") 

57 except LookupError: 

58 raise unittest.SkipTest("obs_decam not set up; needed for ap_pipe_testdata") 

59 

60 def _setupObjPatch(self, *args, **kwargs): 

61 """Create a patch in setUp that will be reverted once the test ends. 

62 

63 Parameters 

64 ---------- 

65 *args 

66 **kwargs 

67 Inputs to `unittest.mock.patch.object`. 

68 """ 

69 patcher = patch.object(*args, **kwargs) 

70 patcher.start() 

71 self.addCleanup(patcher.stop) 

72 

73 def setUp(self): 

74 self.config = self._makeDefaultConfig() 

75 self.butler = dafPersist.Butler(inputs={'root': self.datadir}) 

76 

77 def makeMockDataRef(datasetType, level=None, dataId={}, **rest): 

78 mockDataRef = Mock(dafPersist.ButlerDataRef) 

79 mockDataRef.dataId = dict(dataId, **rest) 

80 return mockDataRef 

81 

82 self._setupObjPatch(self.butler, "dataRef", side_effect=makeMockDataRef) 

83 self.dataId = {"visit": 413635, "ccdnum": 42} 

84 self.inputRef = self.butler.dataRef("raw", **self.dataId) 

85 

86 @contextlib.contextmanager 

87 def mockPatchSubtasks(self, task): 

88 """Make mocks for all the ap_pipe subtasks. 

89 

90 This is needed because the task itself cannot be a mock. 

91 The task's subtasks do not exist until the task is created, so 

92 this allows us to mock them instead. 

93 

94 Parameters 

95 ---------- 

96 task : `lsst.ap.pipe.ApPipeTask` 

97 The task whose subtasks will be mocked. 

98 

99 Yields 

100 ------ 

101 subtasks : `lsst.pipe.base.Struct` 

102 All mocks created by this context manager, including: 

103 

104 ``ccdProcessor`` 

105 ``differencer`` 

106 ``diaPipe`` 

107 a mock for the corresponding subtask. Mocks do not return any 

108 particular value, but have mocked methods that can be queried 

109 for calls by ApPipeTask 

110 """ 

111 with patch.object(task, "ccdProcessor", autospec=True) as mockCcdProcessor, \ 

112 patch.object(task, "differencer", autospec=True) as mockDifferencer, \ 

113 patch.object(task, "transformDiaSrcCat", autospec=True) as mockTransform, \ 

114 patch.object(task, "diaPipe", autospec=True) as mockDiaPipe: 

115 yield pipeBase.Struct(ccdProcessor=mockCcdProcessor, 

116 differencer=mockDifferencer, 

117 transformDiaSrcCat=mockTransform, 

118 diaPipe=mockDiaPipe) 

119 

120 def testGenericRun(self): 

121 """Test the normal workflow of each ap_pipe step. 

122 """ 

123 task = ApPipeTask(self.butler, config=self.config) 

124 with self.mockPatchSubtasks(task) as subtasks: 

125 task.runDataRef(self.inputRef) 

126 subtasks.ccdProcessor.runDataRef.assert_called_once() 

127 subtasks.differencer.runDataRef.assert_called_once() 

128 subtasks.diaPipe.run.assert_called_once() 

129 

130 def testReuseExistingOutput(self): 

131 """Test reuse keyword to ApPipeTask.runDataRef. 

132 """ 

133 task = ApPipeTask(self.butler, config=self.config) 

134 

135 self.checkReuseExistingOutput(task, ['ccdProcessor']) 

136 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer']) 

137 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer', 'diaPipe']) 

138 

139 def checkReuseExistingOutput(self, task, skippable): 

140 """Check whether a task's subtasks are skipped when "reuse" is set. 

141 

142 Mock guarantees that all "has this been made" tests pass, 

143 so skippable subtasks should actually be skipped. 

144 """ 

145 # extremely brittle because it depends on internal code of runDataRef 

146 # but it only needs to work until DM-21886, then we can unit-test the subtask 

147 internalRef = self.inputRef.getButler().dataRef() 

148 internalRef.put.reset_mock() 

149 

150 with self.mockPatchSubtasks(task) as subtasks: 

151 struct = task.runDataRef(self.inputRef, reuse=skippable) 

152 for subtaskName, runner in { 

153 'ccdProcessor': subtasks.ccdProcessor.runDataRef, 

154 'differencer': subtasks.differencer.runDataRef, 

155 'diaPipe': subtasks.diaPipe.run, 

156 }.items(): 

157 msg = "subtask = " + subtaskName 

158 if subtaskName in skippable: 

159 runner.assert_not_called() 

160 self.assertIsNone(struct.getDict()[subtaskName], msg=msg) 

161 else: 

162 runner.assert_called_once() 

163 self.assertIsNotNone(struct.getDict()[subtaskName], msg=msg) 

164 

165 if 'diaPipe' in skippable: 

166 internalRef.put.assert_not_called() 

167 else: 

168 internalRef.put.assert_called_once_with(ANY, "apdb_marker") 

169 

170 def testCalexpRun(self): 

171 """Test the calexp template workflow of each ap_pipe step. 

172 """ 

173 calexpConfigFile = os.path.join(lsst.utils.getPackageDir('ap_pipe'), 

174 'config', 'calexpTemplates.py') 

175 calexpConfig = self._makeDefaultConfig() 

176 calexpConfig.load(calexpConfigFile) 

177 calexpConfig.differencer.doSelectSources = False # Workaround for DM-18394 

178 

179 task = ApPipeTask(self.butler, config=calexpConfig) 

180 with self.mockPatchSubtasks(task) as subtasks: 

181 # We use the same dataId here for both template and science 

182 # in difference imaging. This is OK because everything is a mock 

183 # and we aren't actually doing any image processing. 

184 task.runDataRef(self.inputRef, templateIds=[self.dataId]) 

185 self.assertEqual(subtasks.ccdProcessor.runDataRef.call_count, 2) 

186 subtasks.differencer.runDataRef.assert_called_once() 

187 subtasks.diaPipe.run.assert_called_once() 

188 

189 

190class MemoryTester(lsst.utils.tests.MemoryTestCase): 

191 pass 

192 

193 

194def setup_module(module): 

195 lsst.utils.tests.init() 

196 

197 

198if __name__ == "__main__": 198 ↛ 199line 198 didn't jump to line 199, because the condition on line 198 was never true

199 lsst.utils.tests.init() 

200 unittest.main()