Coverage for tests/test_quickLook.py: 16%

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import contextlib 

23import tempfile 

24import unittest 

25 

26import numpy as np 

27 

28import lsst.afw.cameraGeom.testUtils as afwTestUtils 

29import lsst.afw.image as afwImage 

30from lsst.afw.image import TransmissionCurve 

31import lsst.daf.butler.tests as butlerTests 

32import lsst.ip.isr.isrMock as isrMock 

33import lsst.ip.isr as ipIsr 

34import lsst.pex.exceptions 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.testUtils 

37from lsst.summit.utils.quickLook import QuickLookIsrTask, QuickLookIsrTaskConfig 

38 

39 

40class QuickLookIsrTaskTestCase(unittest.TestCase): 

41 """Tests of the run method with fake data. 

42 """ 

43 

44 def setUp(self): 

45 self.mockConfig = isrMock.IsrMockConfig() 

46 self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig) 

47 self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() 

48 

49 self.ccdExposure = isrMock.RawMock(config=self.mockConfig).run() 

50 self.detector = self.ccdExposure.getDetector() 

51 amps = self.detector.getAmplifiers() 

52 ampNames = [amp.getName() for amp in amps] 

53 

54 # # Mock other optional parameters 

55 self.bias = self.dataContainer.get("bias") 

56 self.dark = self.dataContainer.get("dark") 

57 self.flat = self.dataContainer.get("flat") 

58 self.defects = self.dataContainer.get("defects") 

59 self.ptc = ipIsr.PhotonTransferCurveDataset(ampNames=ampNames) # Mock PTC dataset 

60 self.bfKernel = self.dataContainer.get("bfKernel") 

61 self.newBFKernel = pipeBase.Struct(gain={}) 

62 for amp_i, amp in enumerate(ampNames): 

63 self.newBFKernel.gain[amp] = 0.9 + 0.1*amp_i 

64 self.task = QuickLookIsrTask(config=QuickLookIsrTaskConfig()) 

65 

66 def test_runQuickLook(self): 

67 # Execute the run method with the mock data 

68 result = self.task.run(self.ccdExposure, 

69 camera=self.camera, 

70 bias=self.bias, 

71 dark=self.dark, 

72 flat=self.flat, 

73 defects=self.defects, 

74 linearizer=None, 

75 crosstalk=None, 

76 bfKernel=self.bfKernel, 

77 newBFKernel=self.newBFKernel, 

78 ptc=self.ptc, 

79 crosstalkSources=None 

80 ) 

81 self.assertIsNotNone(result, "Result of run method should not be None") 

82 self.assertIsInstance(result, pipeBase.Struct, "Result should be of type lsst.pipe.base.Struct") 

83 self.assertIsInstance(result.exposure, afwImage.Exposure, 

84 "Resulting exposure should be an instance of lsst.afw.image.Exposure") 

85 

86 def test_runQuickLookMissingData(self): 

87 # Test without any inputs other than the exposure 

88 result = self.task.run(self.ccdExposure) 

89 self.assertIsInstance(result.exposure, afwImage.Exposure) 

90 

91 def test_runQuickLookBadDark(self): 

92 # Test with an incorrect dark frame 

93 bbox = self.ccdExposure.getBBox() 

94 bbox.grow(-20) 

95 with self.assertRaises(lsst.pex.exceptions.wrappers.LengthError): 

96 self.task.run(self.ccdExposure, camera=self.camera, bias=self.bias, 

97 dark=self.dark[bbox], flat=self.flat, defects=self.defects) 

98 

99 

100class QuickLookIsrTaskRunQuantumTests(lsst.utils.tests.TestCase): 

101 """Tests of ``QuickLookIsrTask.runQuantum``, which need a test butler, 

102 but do not need real images. 

103 

104 Adapted from the unit tests of ``CalibrateImageTask.runQuantum`` 

105 """ 

106 def setUp(self): 

107 instrument = "testCam" 

108 exposureId = 100 

109 visit = 100101 

110 detector = 0 

111 physical_filter = "testCam_filter" 

112 band = "X" 

113 

114 # Map the isrTask connection names to the names of the Butler dataset 

115 # inputs 

116 ccdExposure = "raw" 

117 camera = "camera" 

118 bias = "bias" 

119 dark = "dark" 

120 flat = "flat" 

121 defects = "defects" 

122 bfKernel = "bfKernel" 

123 newBFKernel = "brighterFatterKernel" 

124 ptc = "ptc" 

125 filterTransmission = "transmission_filter" 

126 deferredChargeCalib = "cpCtiCalib" 

127 opticsTransmission = "transmission_optics" 

128 strayLightData = "yBackground" 

129 atmosphereTransmission = "transmission_atmosphere" 

130 crosstalk = "crosstalk" 

131 illumMaskedImage = "illum" 

132 linearizer = "linearizer" 

133 fringes = "fringe" 

134 sensorTransmission = "transmission_sensor" 

135 crosstalkSources = "isrOverscanCorrected" 

136 

137 # outputs 

138 outputExposure = "postISRCCD" 

139 

140 # quickLook-only outputs 

141 exposure = "quickLookExp" 

142 

143 # Create a and populate a test butler for runQuantum tests. 

144 self.repo_path = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) 

145 self.repo = butlerTests.makeTestRepo(self.repo_path.name) 

146 

147 # dataIds for fake data 

148 butlerTests.addDataIdValue(self.repo, "instrument", instrument) 

149 butlerTests.addDataIdValue(self.repo, "physical_filter", physical_filter, band=band) 

150 butlerTests.addDataIdValue(self.repo, "detector", detector) 

151 butlerTests.addDataIdValue(self.repo, "exposure", exposureId, physical_filter=physical_filter) 

152 butlerTests.addDataIdValue(self.repo, "visit", visit) 

153 

154 # inputs 

155 butlerTests.addDatasetType(self.repo, ccdExposure, 

156 {"instrument", "exposure", "detector"}, "Exposure") 

157 butlerTests.addDatasetType(self.repo, camera, 

158 {"instrument"}, "Camera") 

159 butlerTests.addDatasetType(self.repo, bias, 

160 {"instrument", "detector"}, "Exposure") 

161 butlerTests.addDatasetType(self.repo, dark, 

162 {"instrument", "detector"}, "Exposure") 

163 butlerTests.addDatasetType(self.repo, flat, 

164 {"instrument", "physical_filter", "detector"}, "Exposure") 

165 butlerTests.addDatasetType(self.repo, defects, 

166 {"instrument", "detector"}, "Defects") 

167 butlerTests.addDatasetType(self.repo, linearizer, 

168 {"instrument", "detector"}, "Linearizer") 

169 butlerTests.addDatasetType(self.repo, crosstalk, 

170 {"instrument", "detector"}, "CrosstalkCalib") 

171 butlerTests.addDatasetType(self.repo, bfKernel, 

172 {"instrument"}, "NumpyArray") 

173 butlerTests.addDatasetType(self.repo, newBFKernel, 

174 {"instrument", "detector"}, "BrighterFatterKernel") 

175 butlerTests.addDatasetType(self.repo, ptc, 

176 {"instrument", "detector"}, "PhotonTransferCurveDataset") 

177 butlerTests.addDatasetType(self.repo, filterTransmission, 

178 {"instrument", "physical_filter"}, "TransmissionCurve") 

179 butlerTests.addDatasetType(self.repo, opticsTransmission, 

180 {"instrument"}, "TransmissionCurve") 

181 butlerTests.addDatasetType(self.repo, deferredChargeCalib, 

182 {"instrument", "detector"}, "IsrCalib") 

183 butlerTests.addDatasetType(self.repo, strayLightData, 

184 {"instrument", "physical_filter", "detector"}, "Exposure") 

185 butlerTests.addDatasetType(self.repo, atmosphereTransmission, 

186 {"instrument"}, "TransmissionCurve") 

187 butlerTests.addDatasetType(self.repo, illumMaskedImage, 

188 {"instrument", "physical_filter", "detector"}, "MaskedImage") 

189 butlerTests.addDatasetType(self.repo, fringes, 

190 {"instrument", "physical_filter", "detector"}, "Exposure") 

191 butlerTests.addDatasetType(self.repo, sensorTransmission, 

192 {"instrument", "detector"}, "TransmissionCurve") 

193 butlerTests.addDatasetType(self.repo, crosstalkSources, 

194 {"instrument", "exposure", "detector"}, "Exposure") 

195 

196 # outputs 

197 butlerTests.addDatasetType(self.repo, outputExposure, 

198 {"instrument", "exposure", "detector"}, "Exposure") 

199 butlerTests.addDatasetType(self.repo, exposure, 

200 {"instrument", "exposure", "detector"}, "Exposure") 

201 

202 # dataIds 

203 self.exposure_id = self.repo.registry.expandDataId( 

204 {"instrument": instrument, "exposure": exposureId, "detector": detector, 

205 "physical_filter": physical_filter}) 

206 self.instrument_id = self.repo.registry.expandDataId( 

207 {"instrument": instrument}) 

208 self.flat_id = self.repo.registry.expandDataId( 

209 {"instrument": instrument, "physical_filter": physical_filter, "detector": detector}) 

210 self.detector_id = self.repo.registry.expandDataId( 

211 {"instrument": instrument, "detector": detector}) 

212 self.filter_id = self.repo.registry.expandDataId( 

213 {"instrument": instrument, "physical_filter": physical_filter}) 

214 

215 # put empty data 

216 transmissionCurve = TransmissionCurve.makeSpatiallyConstant(np.ones(2), np.linspace(0, 1, 2), 0., 0.) 

217 self.butler = butlerTests.makeTestCollection(self.repo) 

218 self.butler.put(afwImage.ExposureF(), ccdExposure, self.exposure_id) 

219 self.butler.put(afwTestUtils.CameraWrapper().camera, camera, self.instrument_id) 

220 self.butler.put(afwImage.ExposureF(), bias, self.detector_id) 

221 self.butler.put(afwImage.ExposureF(), dark, self.detector_id) 

222 self.butler.put(afwImage.ExposureF(), flat, self.flat_id) 

223 self.butler.put(lsst.ip.isr.Defects(), defects, self.detector_id) 

224 self.butler.put(np.zeros(2), bfKernel, self.instrument_id) 

225 self.butler.put(lsst.ip.isr.brighterFatterKernel.BrighterFatterKernel(), 

226 newBFKernel, self.detector_id) 

227 self.butler.put(ipIsr.PhotonTransferCurveDataset(), ptc, self.detector_id) 

228 self.butler.put(transmissionCurve, filterTransmission, self.filter_id) 

229 self.butler.put(lsst.ip.isr.calibType.IsrCalib(), deferredChargeCalib, self.detector_id) 

230 self.butler.put(transmissionCurve, opticsTransmission, self.instrument_id) 

231 self.butler.put(afwImage.ExposureF(), strayLightData, self.flat_id) 

232 self.butler.put(transmissionCurve, atmosphereTransmission, self.instrument_id) 

233 self.butler.put(lsst.ip.isr.crosstalk.CrosstalkCalib(), crosstalk, self.detector_id) 

234 self.butler.put(afwImage.ExposureF().maskedImage, illumMaskedImage, self.flat_id) 

235 self.butler.put(lsst.ip.isr.linearize.Linearizer(), linearizer, self.detector_id) 

236 self.butler.put(afwImage.ExposureF(), fringes, self.flat_id) 

237 self.butler.put(transmissionCurve, sensorTransmission, self.detector_id) 

238 self.butler.put(afwImage.ExposureF(), crosstalkSources, self.exposure_id) 

239 

240 def tearDown(self): 

241 del self.repo_path # this removes the temporary directory 

242 

243 def test_runQuantum(self): 

244 config = ipIsr.IsrTaskConfig() 

245 # Remove some outputs 

246 config.doBinnedExposures = False 

247 config.doSaveInterpPixels = False 

248 config.qa.doThumbnailOss = False 

249 config.qa.doThumbnailFlattened = False 

250 config.doCalculateStatistics = False 

251 

252 # Turn on all optional inputs 

253 config.doAttachTransmissionCurve = True 

254 config.doIlluminationCorrection = True 

255 config.doStrayLight = True 

256 config.doDeferredCharge = True 

257 config.usePtcReadNoise = True 

258 config.doCrosstalk = True 

259 config.doBrighterFatter = True 

260 

261 # Override a method in IsrTask that is executed early, to instead raise 

262 # a custom exception called ExitMock that we can catch and ignore. 

263 isrTask = ipIsr.IsrTask 

264 isrTask.ensureExposure = raiseExitMockError 

265 task = QuickLookIsrTask(isrTask=isrTask) 

266 lsst.pipe.base.testUtils.assertValidInitOutput(task) 

267 

268 # Use the names of the connections here, not the Butler dataset name 

269 quantum = lsst.pipe.base.testUtils.makeQuantum( 

270 task, self.butler, self.exposure_id, 

271 {"ccdExposure": self.exposure_id, 

272 "camera": self.instrument_id, 

273 "bias": self.detector_id, 

274 "dark": self.detector_id, 

275 "flat": self.flat_id, 

276 "defects": self.detector_id, 

277 "bfKernel": self.instrument_id, 

278 "newBFKernel": self.detector_id, 

279 "ptc": self.detector_id, 

280 "filterTransmission": self.filter_id, 

281 "deferredChargeCalib": self.detector_id, 

282 "opticsTransmission": self.instrument_id, 

283 "strayLightData": self.flat_id, 

284 "atmosphereTransmission": self.instrument_id, 

285 "crosstalk": self.detector_id, 

286 "illumMaskedImage": self.flat_id, 

287 "linearizer": self.detector_id, 

288 "fringes": self.flat_id, 

289 "sensorTransmission": self.detector_id, 

290 "crosstalkSources": [self.exposure_id, self.exposure_id], 

291 # outputs 

292 "outputExposure": self.exposure_id, 

293 "exposure": self.exposure_id, 

294 }) 

295 # Check that the proper kwargs are passed to run(). 

296 with contextlib.suppress(ExitMockError): 

297 lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum, mockRun=False) 

298 

299 

300def raiseExitMockError(*args): 

301 """Raise a custom exception. 

302 """ 

303 raise ExitMockError 

304 

305 

306class ExitMockError(Exception): 

307 

308 """A custom exception to catch during a unit test. 

309 """ 

310 

311 pass