Coverage for tests/test_testPipeline.py: 23%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

135 statements  

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import shutil 

25import tempfile 

26import unittest 

27 

28import pandas 

29 

30import lsst.utils.tests 

31import lsst.geom 

32import lsst.afw.image as afwImage 

33import lsst.afw.math as afwMath 

34import lsst.afw.table as afwTable 

35import lsst.skymap 

36import lsst.daf.butler.tests as butlerTests 

37import lsst.pipe.base.testUtils as pipelineTests 

38from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \ 

39 MockCalibrateTask, MockImageDifferenceTask, MockTransformDiaSourceCatalogTask, \ 

40 MockDiaPipelineTask 

41 

42 

43class MockTaskTestSuite(unittest.TestCase): 

44 """Test that mock tasks have the correct inputs and outputs for the task 

45 they are replacing. 

46 

47 These tests assume that the mock tasks use real config and connection 

48 classes, and therefore out-of-date mocks won't match their connections. 

49 """ 

50 

51 @classmethod 

52 def setUpClass(cls): 

53 super().setUpClass() 

54 

55 repoDir = tempfile.mkdtemp() 

56 cls.addClassCleanup(shutil.rmtree, repoDir, ignore_errors=True) 

57 cls.repo = butlerTests.makeTestRepo(repoDir) 

58 

59 INSTRUMENT = "notACam" 

60 VISIT = 42 

61 CCD = 101 

62 HTM = 42 

63 SKYMAP = "TreasureMap" 

64 TRACT = 28 

65 PATCH = 4 

66 BAND = 'k' 

67 PHYSICAL = 'k2022' 

68 SUB_FILTER = 9 

69 # Mock instrument by hand, because some tasks care about parameters 

70 instrumentRecord = cls.repo.registry.dimensions["instrument"].RecordClass( 

71 name=INSTRUMENT, visit_max=256, exposure_max=256, detector_max=128) 

72 cls.repo.registry.syncDimensionData("instrument", instrumentRecord) 

73 butlerTests.addDataIdValue(cls.repo, "physical_filter", PHYSICAL, band=BAND) 

74 butlerTests.addDataIdValue(cls.repo, "subfilter", SUB_FILTER) 

75 butlerTests.addDataIdValue(cls.repo, "exposure", VISIT) 

76 butlerTests.addDataIdValue(cls.repo, "visit", VISIT) 

77 butlerTests.addDataIdValue(cls.repo, "detector", CCD) 

78 butlerTests.addDataIdValue(cls.repo, "skymap", SKYMAP) 

79 butlerTests.addDataIdValue(cls.repo, "tract", TRACT) 

80 butlerTests.addDataIdValue(cls.repo, "patch", PATCH) 

81 

82 cls.exposureId = cls.repo.registry.expandDataId( 

83 {"instrument": INSTRUMENT, "exposure": VISIT, "detector": CCD}) 

84 cls.visitId = cls.repo.registry.expandDataId( 

85 {"instrument": INSTRUMENT, "visit": VISIT, "detector": CCD}) 

86 cls.visitOnlyId = cls.repo.registry.expandDataId( 

87 {"instrument": INSTRUMENT, "visit": VISIT}) 

88 cls.skymapId = cls.repo.registry.expandDataId({"skymap": SKYMAP}) 

89 cls.skymapVisitId = cls.repo.registry.expandDataId( 

90 {"instrument": INSTRUMENT, "visit": VISIT, "detector": CCD, "skymap": SKYMAP}) 

91 cls.patchId = cls.repo.registry.expandDataId( 

92 {"skymap": SKYMAP, "tract": TRACT, "patch": PATCH, "band": BAND}) 

93 cls.subfilterId = cls.repo.registry.expandDataId( 

94 {"skymap": SKYMAP, "tract": TRACT, "patch": PATCH, "band": BAND, "subfilter": SUB_FILTER}) 

95 cls.htmId = cls.repo.registry.expandDataId({"htm7": HTM}) 

96 

97 butlerTests.addDatasetType(cls.repo, "postISRCCD", cls.exposureId.keys(), "Exposure") 

98 butlerTests.addDatasetType(cls.repo, "icExp", cls.visitId.keys(), "ExposureF") 

99 butlerTests.addDatasetType(cls.repo, "icSrc", cls.visitId.keys(), "SourceCatalog") 

100 butlerTests.addDatasetType(cls.repo, "icExpBackground", cls.visitId.keys(), "Background") 

101 butlerTests.addDatasetType(cls.repo, "cal_ref_cat", cls.htmId.keys(), "SimpleCatalog") 

102 butlerTests.addDatasetType(cls.repo, "calexp", cls.visitId.keys(), "ExposureF") 

103 butlerTests.addDatasetType(cls.repo, "src", cls.visitId.keys(), "SourceCatalog") 

104 butlerTests.addDatasetType(cls.repo, "calexpBackground", cls.visitId.keys(), "Background") 

105 butlerTests.addDatasetType(cls.repo, "srcMatch", cls.visitId.keys(), "Catalog") 

106 butlerTests.addDatasetType(cls.repo, "srcMatchFull", cls.visitId.keys(), "Catalog") 

107 butlerTests.addDatasetType(cls.repo, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

108 cls.skymapId.keys(), "SkyMap") 

109 butlerTests.addDatasetType(cls.repo, "deepCoadd", cls.patchId.keys(), "ExposureF") 

110 butlerTests.addDatasetType(cls.repo, "dcrCoadd", cls.subfilterId.keys(), "ExposureF") 

111 butlerTests.addDatasetType(cls.repo, "deepDiff_differenceExp", cls.visitId.keys(), "ExposureF") 

112 butlerTests.addDatasetType(cls.repo, "deepDiff_scoreExp", cls.visitId.keys(), "ExposureF") 

113 butlerTests.addDatasetType(cls.repo, "deepDiff_warpedExp", cls.visitId.keys(), "ExposureF") 

114 butlerTests.addDatasetType(cls.repo, "deepDiff_matchedExp", cls.visitId.keys(), "ExposureF") 

115 butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrc", cls.visitId.keys(), "SourceCatalog") 

116 butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrcTable", cls.visitId.keys(), "DataFrame") 

117 butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.keys(), "DataFrame") 

118 butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.keys(), "Config") 

119 butlerTests.addDatasetType(cls.repo, "deepDiff_associDiaSrc", cls.visitId.keys(), "DataFrame") 

120 

121 def setUp(self): 

122 super().setUp() 

123 self.butler = butlerTests.makeTestCollection(self.repo, uniqueId=self.id()) 

124 

125 def testMockIsr(self): 

126 # Testing MockIsrTask is tricky because the real ISR has an unstable 

127 # interface with dozens of potential inputs, too many to pass through 

128 # runTestQuantum. I don't see a good way to test the inputs; 

129 # fortunately, this is unlikely to matter for the overall goal of 

130 # testing ap_verify's interaction with the AP pipeline. 

131 task = MockIsrTask() 

132 pipelineTests.assertValidInitOutput(task) 

133 result = task.run(afwImage.ExposureF()) 

134 pipelineTests.assertValidOutput(task, result) 

135 # Skip runTestQuantum 

136 

137 def testMockCharacterizeImageTask(self): 

138 task = MockCharacterizeImageTask() 

139 pipelineTests.assertValidInitOutput(task) 

140 result = task.run(afwImage.ExposureF()) 

141 pipelineTests.assertValidOutput(task, result) 

142 

143 self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposureId) 

144 quantum = pipelineTests.makeQuantum( 

145 task, self.butler, self.visitId, 

146 {"exposure": self.exposureId, 

147 "characterized": self.visitId, 

148 "sourceCat": self.visitId, 

149 "backgroundModel": self.visitId, 

150 }) 

151 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) 

152 

153 def testMockCalibrateTask(self): 

154 task = MockCalibrateTask() 

155 pipelineTests.assertValidInitOutput(task) 

156 # Even the real CalibrateTask won't pass assertValidOutput, because for 

157 # some reason the outputs are injected in runQuantum rather than run. 

158 

159 self.butler.put(afwImage.ExposureF(), "icExp", self.visitId) 

160 self.butler.put(afwMath.BackgroundList(), "icExpBackground", self.visitId) 

161 self.butler.put(afwTable.SourceCatalog(), "icSrc", self.visitId) 

162 self.butler.put(afwTable.SimpleCatalog(), "cal_ref_cat", self.htmId) 

163 quantum = pipelineTests.makeQuantum( 

164 task, self.butler, self.visitId, 

165 {"exposure": self.visitId, 

166 "background": self.visitId, 

167 "icSourceCat": self.visitId, 

168 "astromRefCat": [self.htmId], 

169 "photoRefCat": [self.htmId], 

170 "outputExposure": self.visitId, 

171 "outputCat": self.visitId, 

172 "outputBackground": self.visitId, 

173 "matches": self.visitId, 

174 "matchesDenormalized": self.visitId, 

175 }) 

176 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) 

177 

178 def testMockImageDifferenceTask(self): 

179 task = MockImageDifferenceTask() 

180 pipelineTests.assertValidInitOutput(task) 

181 result = task.run(afwImage.ExposureF(), templateExposure=afwImage.ExposureF()) 

182 pipelineTests.assertValidOutput(task, result) 

183 

184 self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) 

185 skymap = lsst.skymap.DiscreteSkyMap(lsst.skymap.DiscreteSkyMapConfig()) 

186 self.butler.put(skymap, lsst.skymap.BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, self.skymapId) 

187 self.butler.put(afwImage.ExposureF(), "deepCoadd", self.patchId) 

188 self.butler.put(afwImage.ExposureF(), "dcrCoadd", self.subfilterId) 

189 quantum = pipelineTests.makeQuantum( 

190 task, self.butler, self.skymapVisitId, 

191 {"exposure": self.visitId, 

192 "skyMap": self.skymapId, 

193 "coaddExposures": [self.patchId], 

194 "dcrCoadds": [self.subfilterId], 

195 "subtractedExposure": self.visitId, 

196 "scoreExposure": self.visitId, 

197 "warpedExposure": self.visitId, 

198 "matchedExposure": self.visitId, 

199 "diaSources": self.visitId, 

200 }) 

201 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) 

202 

203 def testMockTransformDiaSourceCatalogTask(self): 

204 task = MockTransformDiaSourceCatalogTask(initInputs=afwTable.SourceCatalog()) 

205 pipelineTests.assertValidInitOutput(task) 

206 result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42) 

207 pipelineTests.assertValidOutput(task, result) 

208 

209 self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId) 

210 self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) 

211 quantum = pipelineTests.makeQuantum( 

212 task, self.butler, self.visitId, 

213 {"diaSourceCat": self.visitId, 

214 "diffIm": self.visitId, 

215 "diaSourceTable": self.visitId, 

216 }) 

217 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) 

218 

219 def testMockDiaPipelineTask(self): 

220 task = MockDiaPipelineTask() 

221 pipelineTests.assertValidInitOutput(task) 

222 result = task.run(pandas.DataFrame(), pandas.DataFrame(), afwImage.ExposureF(), 

223 afwImage.ExposureF(), afwImage.ExposureF(), 42, 'k') 

224 pipelineTests.assertValidOutput(task, result) 

225 

226 self.butler.put(pandas.DataFrame(), "deepDiff_diaSrcTable", self.visitId) 

227 self.butler.put(pandas.DataFrame(), "visitSsObjects", self.visitId) 

228 self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId) 

229 self.butler.put(afwImage.ExposureF(), "calexp", self.visitId) 

230 self.butler.put(afwImage.ExposureF(), "deepDiff_warpedExp", self.visitId) 

231 quantum = pipelineTests.makeQuantum( 

232 task, self.butler, self.visitId, 

233 {"diaSourceTable": self.visitId, 

234 "solarSystemObjectTable": self.visitId, 

235 "diffIm": self.visitId, 

236 "exposure": self.visitId, 

237 "warpedExposure": self.visitId, 

238 "apdbMarker": self.visitId, 

239 "associatedDiaSources": self.visitId, 

240 }) 

241 pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False) 

242 

243 

244class MemoryTester(lsst.utils.tests.MemoryTestCase): 

245 pass 

246 

247 

248def setup_module(module): 

249 lsst.utils.tests.init() 

250 

251 

252if __name__ == "__main__": 252 ↛ 253line 252 didn't jump to line 253, because the condition on line 252 was never true

253 lsst.utils.tests.init() 

254 unittest.main()