Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_pipe. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import contextlib 

25import os 

26import unittest 

27from unittest.mock import patch, Mock, ANY 

28 

29import lsst.utils.tests 

30import lsst.pex.exceptions as pexExcept 

31import lsst.daf.persistence as dafPersist 

32import lsst.pipe.base as pipeBase 

33 

34from lsst.ap.pipe import ApPipeTask 

35 

36 

37class PipelineTestSuite(lsst.utils.tests.TestCase): 

38 ''' 

39 A set of tests for the functions in ap_pipe. 

40 ''' 

41 

42 @classmethod 

43 def _makeDefaultConfig(cls): 

44 config = ApPipeTask.ConfigClass() 

45 config.load(os.path.join(cls.datadir, "config", "apPipe.py")) 

46 config.diaPipe.apdb.db_url = "sqlite://" 

47 config.diaPipe.apdb.isolation_level = "READ_UNCOMMITTED" 

48 return config 

49 

50 @classmethod 

51 def setUpClass(cls): 

52 try: 

53 cls.datadir = lsst.utils.getPackageDir("ap_pipe_testdata") 

54 except pexExcept.NotFoundError: 

55 raise unittest.SkipTest("ap_pipe_testdata not set up") 

56 

57 def _setupObjPatch(self, *args, **kwargs): 

58 """Create a patch in setUp that will be reverted once the test ends. 

59 

60 Parameters 

61 ---------- 

62 *args 

63 **kwargs 

64 Inputs to `unittest.mock.patch.object`. 

65 """ 

66 patcher = patch.object(*args, **kwargs) 

67 patcher.start() 

68 self.addCleanup(patcher.stop) 

69 

70 def setUp(self): 

71 self.config = self._makeDefaultConfig() 

72 self.butler = dafPersist.Butler(inputs={'root': self.datadir}) 

73 

74 def makeMockDataRef(datasetType, level=None, dataId={}, **rest): 

75 mockDataRef = Mock(dafPersist.ButlerDataRef) 

76 mockDataRef.dataId = dict(dataId, **rest) 

77 return mockDataRef 

78 

79 self._setupObjPatch(self.butler, "dataRef", side_effect=makeMockDataRef) 

80 self.dataId = {"visit": 413635, "ccdnum": 42} 

81 self.inputRef = self.butler.dataRef("raw", **self.dataId) 

82 

83 @contextlib.contextmanager 

84 def mockPatchSubtasks(self, task): 

85 """Make mocks for all the ap_pipe subtasks. 

86 

87 This is needed because the task itself cannot be a mock. 

88 The task's subtasks do not exist until the task is created, so 

89 this allows us to mock them instead. 

90 

91 Parameters 

92 ---------- 

93 task : `lsst.ap.pipe.ApPipeTask` 

94 The task whose subtasks will be mocked. 

95 

96 Yields 

97 ------ 

98 subtasks : `lsst.pipe.base.Struct` 

99 All mocks created by this context manager, including: 

100 

101 ``ccdProcessor`` 

102 ``differencer`` 

103 ``diaPipe`` 

104 a mock for the corresponding subtask. Mocks do not return any 

105 particular value, but have mocked methods that can be queried 

106 for calls by ApPipeTask 

107 """ 

108 with patch.object(task, "ccdProcessor") as mockCcdProcessor, \ 

109 patch.object(task, "differencer") as mockDifferencer, \ 

110 patch.object(task, "diaPipe") as mockDiaPipe: 

111 yield pipeBase.Struct(ccdProcessor=mockCcdProcessor, 

112 differencer=mockDifferencer, 

113 diaPipe=mockDiaPipe) 

114 

115 def testGenericRun(self): 

116 """Test the normal workflow of each ap_pipe step. 

117 """ 

118 task = ApPipeTask(self.butler, config=self.config) 

119 with self.mockPatchSubtasks(task) as subtasks: 

120 task.runDataRef(self.inputRef) 

121 subtasks.ccdProcessor.runDataRef.assert_called_once() 

122 subtasks.differencer.runDataRef.assert_called_once() 

123 subtasks.diaPipe.run.assert_called_once() 

124 

125 def testReuseExistingOutput(self): 

126 """Test reuse keyword to ApPipeTask.runDataRef. 

127 """ 

128 task = ApPipeTask(self.butler, config=self.config) 

129 

130 self.checkReuseExistingOutput(task, ['ccdProcessor']) 

131 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer']) 

132 self.checkReuseExistingOutput(task, ['ccdProcessor', 'differencer', 'diaPipe']) 

133 

134 def checkReuseExistingOutput(self, task, skippable): 

135 """Check whether a task's subtasks are skipped when "reuse" is set. 

136 

137 Mock guarantees that all "has this been made" tests pass, 

138 so skippable subtasks should actually be skipped. 

139 """ 

140 # extremely brittle because it depends on internal code of runDataRef 

141 # but it only needs to work until DM-21886, then we can unit-test the subtask 

142 internalRef = self.inputRef.getButler().dataRef() 

143 internalRef.put.reset_mock() 

144 

145 with self.mockPatchSubtasks(task) as subtasks: 

146 struct = task.runDataRef(self.inputRef, reuse=skippable) 

147 for subtaskName, runner in { 

148 'ccdProcessor': subtasks.ccdProcessor.runDataRef, 

149 'differencer': subtasks.differencer.runDataRef, 

150 'diaPipe': subtasks.diaPipe.run, 

151 }.items(): 

152 msg = "subtask = " + subtaskName 

153 if subtaskName in skippable: 

154 runner.assert_not_called() 

155 self.assertIsNone(struct.getDict()[subtaskName], msg=msg) 

156 else: 

157 runner.assert_called_once() 

158 self.assertIsNotNone(struct.getDict()[subtaskName], msg=msg) 

159 

160 if 'diaPipe' in skippable: 

161 internalRef.put.assert_not_called() 

162 else: 

163 internalRef.put.assert_called_once_with(ANY, "apdb_marker") 

164 

165 def testCalexpRun(self): 

166 """Test the calexp template workflow of each ap_pipe step. 

167 """ 

168 calexpConfigFile = os.path.join(lsst.utils.getPackageDir('ap_pipe'), 

169 'config', 'calexpTemplates.py') 

170 calexpConfig = self._makeDefaultConfig() 

171 calexpConfig.load(calexpConfigFile) 

172 calexpConfig.differencer.doSelectSources = False # Workaround for DM-18394 

173 

174 task = ApPipeTask(self.butler, config=calexpConfig) 

175 with self.mockPatchSubtasks(task) as subtasks: 

176 # We use the same dataId here for both template and science 

177 # in difference imaging. This is OK because everything is a mock 

178 # and we aren't actually doing any image processing. 

179 task.runDataRef(self.inputRef, templateIds=[self.dataId]) 

180 self.assertEqual(subtasks.ccdProcessor.runDataRef.call_count, 2) 

181 subtasks.differencer.runDataRef.assert_called_once() 

182 subtasks.diaPipe.run.assert_called_once() 

183 

184 

185class MemoryTester(lsst.utils.tests.MemoryTestCase): 

186 pass 

187 

188 

189def setup_module(module): 

190 lsst.utils.tests.init() 

191 

192 

193if __name__ == "__main__": 193 ↛ 194line 193 didn't jump to line 194, because the condition on line 193 was never true

194 lsst.utils.tests.init() 

195 unittest.main()