Coverage for tests/test_calibrate.py: 28%

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Test ProcessCcdTask and its immediate subtasks. 

23""" 

24import logging 

25import os 

26import shutil 

27import tempfile 

28import unittest 

29 

30import lsst.utils.tests 

31import lsst.afw.image 

32import lsst.afw.math 

33import lsst.afw.table 

34import lsst.daf.butler.tests as butlerTests 

35from lsst.pipe.base import testUtils 

36from lsst.pipe.tasks.calibrate import CalibrateTask, CalibrateConfig 

37from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask, CharacterizeImageConfig 

38import lsst.meas.extensions.piff.piffPsfDeterminer 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43class CalibrateTaskTestCaseWithButler(lsst.utils.tests.TestCase): 

44 

45 @classmethod 

46 def _makeTestRepo(cls, root): 

47 """Create a repository with the metadata assumed by CalibrateTask. 

48 """ 

49 # In-memory for performance 

50 config = lsst.daf.butler.Config() 

51 config["datastore", "cls"] = "lsst.daf.butler.datastores.inMemoryDatastore.InMemoryDatastore" 

52 config["datastore", "checksum"] = False 

53 config["registry", "db"] = "sqlite:///:memory:" 

54 

55 butler = lsst.daf.butler.Butler(lsst.daf.butler.Butler.makeRepo(root, config=config), writeable=True) 

56 butler.registry.insertDimensionData( 

57 "instrument", 

58 {"name": "notACam", "visit_max": 256, "exposure_max": 256, "detector_max": 64}) 

59 butler.registry.insertDimensionData( 

60 "physical_filter", 

61 {"instrument": "notACam", "name": "r", "band": "r"}, 

62 ) 

63 if "day_obs" in butler.dimensions: 

64 butler.registry.insertDimensionData( 

65 "day_obs", 

66 {"id": 20240201, "instrument": "notACam"}, 

67 ) 

68 butler.registry.insertDimensionData( 

69 "visit", 

70 {"instrument": "notACam", "id": 101, "name": "101", "physical_filter": "r", "day_obs": 20240201}, 

71 ) 

72 butler.registry.insertDimensionData("detector", 

73 {"instrument": "notACam", "id": 42, "full_name": "42"}) 

74 return butler 

75 

76 @classmethod 

77 def setUpClass(cls): 

78 super().setUpClass() 

79 

80 cls.root = tempfile.mkdtemp() 

81 cls.repo = cls._makeTestRepo(cls.root) 

82 

83 butlerTests.addDatasetType( 

84 cls.repo, "icExp", {"instrument", "visit", "detector"}, 

85 "ExposureF") 

86 butlerTests.addDatasetType( 

87 cls.repo, "icExpBackground", {"instrument", "visit", "detector"}, 

88 "Background") 

89 butlerTests.addDatasetType( 

90 cls.repo, "icSrc", {"instrument", "visit", "detector"}, 

91 "SourceCatalog") 

92 butlerTests.addDatasetType( 

93 cls.repo, "cal_ref_cat", {"htm7"}, 

94 "SimpleCatalog") 

95 butlerTests.addDatasetType( 

96 cls.repo, "calexp", {"instrument", "visit", "detector"}, 

97 "ExposureF") 

98 butlerTests.addDatasetType( 

99 cls.repo, "src", {"instrument", "visit", "detector"}, 

100 "SourceCatalog") 

101 butlerTests.addDatasetType( 

102 cls.repo, "calexpBackground", {"instrument", "visit", "detector"}, 

103 "Background") 

104 butlerTests.addDatasetType( 

105 cls.repo, "srcMatch", {"instrument", "visit", "detector"}, 

106 "Catalog") 

107 butlerTests.addDatasetType( 

108 cls.repo, "srcMatchFull", {"instrument", "visit", "detector"}, 

109 "Catalog") 

110 

111 @classmethod 

112 def tearDownClass(cls): 

113 shutil.rmtree(cls.root, ignore_errors=True) 

114 super().tearDownClass() 

115 

116 def setUp(self): 

117 super().setUp() 

118 self.butler = butlerTests.makeTestCollection(self.repo, uniqueId=self.id()) 

119 

120 self.dataId = {"instrument": "notACam", "visit": 101, "detector": 42} 

121 # CalibrateTask absolutely requires an ExpandedDataCoordinate 

122 self.dataId = self.butler.registry.expandDataId(self.dataId) 

123 self.refcatId = {"htm7": 189584} 

124 

125 # Tests do no processing, so we don't need real data 

126 self.exposure = lsst.afw.image.ExposureF(10, 10) 

127 background = lsst.afw.math.BackgroundMI(self.exposure.getBBox(), self.exposure.getMaskedImage()) 

128 self.backgroundlist = lsst.afw.math.BackgroundList( 

129 (background, lsst.afw.math.Interpolate.UNKNOWN, lsst.afw.math.UndersampleStyle.THROW_EXCEPTION, 

130 lsst.afw.math.ApproximateControl.UNKNOWN, 0, 0, 1)) 

131 self.icSrc = lsst.afw.table.SourceCatalog() 

132 self.refcat = lsst.afw.table.SimpleCatalog() 

133 

134 self.butler.put(self.exposure, "icExp", self.dataId) 

135 self.butler.put(self.backgroundlist, "icExpBackground", self.dataId) 

136 self.butler.put(self.icSrc, "icSrc", self.dataId) 

137 self.butler.put(self.refcat, "cal_ref_cat", self.refcatId) 

138 

139 def testDoAstrometry(self): 

140 """Ensure correct inputs passed to run whether or not doAstrometry 

141 is set. 

142 """ 

143 allIds = {key: self.dataId for key in { 

144 "exposure", "background", "icSourceCat", "outputExposure", "outputCat", "outputBackground", 

145 "matches", "matchesDenormalized" 

146 }} 

147 allIds.update({key: [self.refcatId] for key in {"astromRefCat", "photoRefCat"}}) 

148 

149 self._checkDoRefcats(doAstrometry=True, doPhotoCal=True, ids=allIds) 

150 self._checkDoRefcats(doAstrometry=False, doPhotoCal=True, ids=allIds) 

151 

152 def testDoPhotoCal(self): 

153 """Ensure correct inputs passed to run whether or not doPhotoCal 

154 is set. 

155 """ 

156 allIds = {key: self.dataId for key in { 

157 "exposure", "background", "icSourceCat", "outputExposure", "outputCat", "outputBackground", 

158 "matches", "matchesDenormalized" 

159 }} 

160 allIds.update({key: [self.refcatId] for key in {"astromRefCat", "photoRefCat"}}) 

161 

162 self._checkDoRefcats(doAstrometry=True, doPhotoCal=True, ids=allIds) 

163 self._checkDoRefcats(doAstrometry=True, doPhotoCal=False, ids=allIds) 

164 

165 def _checkDoRefcats(self, doAstrometry, doPhotoCal, ids): 

166 """Test whether run is called with the correct arguments. 

167 

168 In the case of `CalibrateTask`, the inputs should not depend on the 

169 task configuration. 

170 

171 Parameters 

172 ---------- 

173 doAstrometry, doPhotoCal : `bool` 

174 Values of the config flags of the same name. 

175 ids : `dict` [`str`] 

176 A mapping from the input dataset type to the data ID of the 

177 dataset to process. 

178 """ 

179 config = CalibrateConfig() 

180 config.doWriteMatches = False # no real output to write 

181 config.doAstrometry = doAstrometry 

182 config.doPhotoCal = doPhotoCal 

183 config.connections.photoRefCat = "cal_ref_cat" 

184 config.connections.astromRefCat = "cal_ref_cat" 

185 config.idGenerator.packer.name = "observation" 

186 task = CalibrateTask(config=config) 

187 quantumId = ids["exposure"] 

188 

189 quantum = testUtils.makeQuantum(task, self.butler, quantumId, ids) 

190 run = testUtils.runTestQuantum(task, self.butler, quantum) 

191 

192 run.assert_called_once() 

193 self.assertEqual(run.call_args[0], ()) 

194 # Some arguments unprintable because we don't have a full environment 

195 # So just check which ones were passed in 

196 self.assertEqual(run.call_args[1].keys(), 

197 {"exposure", "idGenerator", "background", "icSourceCat"}) 

198 

199 

200class CalibrateTaskTestCaseNoButler(lsst.utils.tests.TestCase): 

201 

202 def testNoAperCorrMap(self): 

203 expPath = os.path.join(TESTDIR, "data", "v695833-e0-c000-a00.sci.fits") 

204 exposure = lsst.afw.image.ExposureF(expPath) 

205 

206 charImConfig = CharacterizeImageConfig() 

207 charImConfig.measurePsf.psfDeterminer = 'piff' 

208 charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0 

209 charImConfig.measureApCorr.sourceSelector["science"].doSignalToNoise = False 

210 charImTask = CharacterizeImageTask(config=charImConfig) 

211 charImResults = charImTask.run(exposure) 

212 calibConfig = CalibrateConfig() 

213 calibConfig.doAstrometry = False 

214 calibConfig.doPhotoCal = False 

215 calibConfig.doSkySources = False 

216 calibConfig.doComputeSummaryStats = False 

217 

218 # Force the aperture correction map to None (DM-39626) 

219 charImResults.exposure.info.setApCorrMap(None) 

220 calibTask = CalibrateTask(config=calibConfig) 

221 with self.assertLogs(level=logging.WARNING) as cm: 

222 _ = calibTask.run(charImResults.exposure) 

223 # Other warnings may also be emitted. 

224 warnings = '\n'.join(cm.output) 

225 self.assertIn("Image does not have valid aperture correction map", warnings) 

226 

227 

228def setup_module(module): 

229 lsst.utils.tests.init() 

230 

231 

232class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

233 pass 

234 

235 

236if __name__ == "__main__": 236 ↛ 237line 236 didn't jump to line 237 because the condition on line 236 was never true

237 lsst.utils.tests.init() 

238 unittest.main()