Coverage for tests / test_fgcmcal_hsc.py: 21%

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Test the fgcmcal code with testdata_jointcal/hsc. 

24 

25Run test suite on fgcmcal using Gen3 HSC data from testdata_jointcal. 

26""" 

27import unittest 

28import os 

29import tempfile 

30import numpy as np 

31 

32# Ensure that matplotlib doesn't try to open a display during testing. 

33import matplotlib 

34matplotlib.use("Agg") 

35 

36import lsst.utils # noqa: E402 

37import lsst.pipe.tasks # noqa: E402 

38import lsst.daf.butler.cli.cliLog # noqa: E402 

39 

40import fgcmcalTestBase # noqa: E402 

41 

42 

43ROOT = os.path.abspath(os.path.dirname(__file__)) 

44 

45I0STD = [0.08294534, 0.07877351, 0.06464688] 

46I10STD = [-0.000091981, -0.00061516, -0.00063434] 

47I0RECON = [0.07322179342588758, 0.0689530429, 0.05600673] 

48I10RECON = [-4.490243571049125, -7.01786443508, 3.62738180611] 

49 

50 

51class FgcmcalTestHSC(fgcmcalTestBase.FgcmcalTestBase, lsst.utils.tests.TestCase): 

52 @classmethod 

53 def setUpClass(cls): 

54 try: 

55 cls.dataDir = lsst.utils.getPackageDir('testdata_jointcal') 

56 except LookupError: 

57 raise unittest.SkipTest("testdata_jointcal not setup") 

58 try: 

59 lsst.utils.getPackageDir('obs_subaru') 

60 except LookupError: 

61 raise unittest.SkipTest("obs_subaru not setup") 

62 

63 lsst.daf.butler.cli.cliLog.CliLog.initLog(longlog=False) 

64 

65 cls.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestFgcm-") 

66 

67 cls._importRepository('lsst.obs.subaru.HyperSuprimeCam', 

68 os.path.join(cls.dataDir, 'hsc/repo'), 

69 os.path.join(cls.dataDir, 'hsc', 'exports.yaml')) 

70 

71 def test_fgcmcalPipelineBuildFromTable(self): 

72 """Test running the full pipeline, using older association code. 

73 

74 This test uses the FgcmBuildStarsFromTableTask instead of the new 

75 FgcmBuildFromIsolatedStarsTask. 

76 """ 

77 # Set numpy seed for stability 

78 np.random.seed(seed=1000) 

79 

80 instName = 'HSC' 

81 testName = 'testfgcmcalpipe' 

82 

83 nBand = 3 

84 i0Std = np.array(I0STD) 

85 i10Std = np.array(I10STD) 

86 i0Recon = np.array(I0RECON) 

87 i10Recon = np.array(I10RECON) 

88 

89 self._testFgcmMakeLut(instName, testName, 

90 nBand, i0Std, i0Recon, i10Std, i10Recon) 

91 

92 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

93 

94 nStar = 305 

95 nObs = 3789 

96 

97 self._testFgcmBuildStarsTable(instName, testName, 

98 "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I')", 

99 visits, nStar, nObs) 

100 

101 nZp = 1120 

102 nGoodZp = 123 

103 nOkZp = 123 

104 nBadZp = 997 

105 nStdStars = 237 

106 nPlots = 66 

107 

108 # We need an extra config file to turn off parquet format. 

109 extraConfigFile = os.path.join(self.testDir, "turn_off_parquet.py") 

110 with open(extraConfigFile, "w") as f: 

111 f.write("config.useParquetCatalogFormat = False\n") 

112 

113 self._testFgcmFitCycle(instName, testName, 

114 0, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True, 

115 extraConfig=extraConfigFile) 

116 self._testFgcmFitCycle(instName, testName, 

117 1, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True, 

118 extraConfig=extraConfigFile) 

119 

120 # We need to create an extra config file to turn on "sub-ccd gray" for testing. 

121 # We also want to exercise the code path setting useExposureReferenceOffset = False. 

122 extraConfigFile = os.path.join(self.testDir, "cycle03_patch_config.py") 

123 with open(extraConfigFile, "w") as f: 

124 f.write("config.useParquetCatalogFormat = False\n") 

125 f.write("config.isFinalCycle = True\n") 

126 f.write("config.ccdGraySubCcdDict = {'g': True, 'r': True, 'i': True}\n") 

127 f.write("config.useExposureReferenceOffset = False") 

128 

129 self._testFgcmFitCycle(instName, testName, 

130 2, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, 

131 extraConfig=extraConfigFile) 

132 

133 def test_fgcmcalPipeline(self): 

134 """Test running the full pipeline, using new isolated star association code. 

135 

136 This test uses the FgcmBuildFromIsolatedStarsTask instead of the old 

137 FgcmBuildStarsFromTableTask. 

138 """ 

139 # Set numpy seed for stability 

140 np.random.seed(seed=1000) 

141 

142 instName = 'HSC' 

143 testName = 'testfgcmcalpipe' 

144 

145 nBand = 3 

146 i0Std = np.array(I0STD) 

147 i10Std = np.array(I10STD) 

148 i0Recon = np.array(I0RECON) 

149 i10Recon = np.array(I10RECON) 

150 

151 self._testFgcmMakeLut(instName, testName, 

152 nBand, i0Std, i0Recon, i10Std, i10Recon) 

153 

154 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

155 

156 nStar = 292 

157 nObs = 1790 

158 

159 self._testFgcmBuildFromIsolatedStars( 

160 instName, 

161 testName, 

162 "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I')", 

163 visits, 

164 nStar, 

165 nObs, 

166 ) 

167 

168 nZp = 1120 

169 nGoodZp = 123 

170 nOkZp = 123 

171 nBadZp = 997 

172 nStdStars = 222 

173 nPlots = 66 

174 

175 self._testFgcmFitCycle(instName, testName, 

176 0, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True) 

177 self._testFgcmFitCycle(instName, testName, 

178 1, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True) 

179 

180 # We need to create an extra config file to turn on "sub-ccd gray" for testing. 

181 # We also want to exercise the code path setting useExposureReferenceOffset = False. 

182 extraConfigFile = os.path.join(self.testDir, "cycle03_patch_config.py") 

183 with open(extraConfigFile, "w") as f: 

184 f.write("config.isFinalCycle = True\n") 

185 f.write("config.ccdGraySubCcdDict = {'g': True, 'r': True, 'i': True}\n") 

186 f.write("config.useExposureReferenceOffset = False") 

187 

188 self._testFgcmFitCycle(instName, testName, 

189 2, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, 

190 extraConfig=extraConfigFile) 

191 

192 zpOffsets = np.array([-0.000842530163936317, 

193 0.0047900681383907795]) 

194 

195 self._testFgcmOutputProducts(instName, testName, 

196 zpOffsets, 36236, 87, 'i', 1, 'hsc_rings_v1') 

197 

198 # Test a single detector illumination correction. 

199 self._testFgcmOutputIlluminationCorrection(instName, testName, 51) 

200 

201 def test_fgcmcalMultipleFitPipeline(self): 

202 # Set numpy seed for stability 

203 np.random.seed(seed=1000) 

204 

205 instName = 'HSC' 

206 testName = 'testfgcmcalmultiple' 

207 

208 nBand = 3 

209 i0Std = np.array(I0STD) 

210 i10Std = np.array(I10STD) 

211 i0Recon = np.array(I0RECON) 

212 i10Recon = np.array(I10RECON) 

213 

214 self._testFgcmMakeLut(instName, testName, 

215 nBand, i0Std, i0Recon, i10Std, i10Recon) 

216 

217 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

218 

219 # These are slightly different from above due to the configuration change 

220 # mid-way in the separate fits. 

221 zpOffsets = np.array([0.0011443052208051085, 

222 0.0015734810149297118]) 

223 

224 self._testFgcmMultiFit(instName, testName, 

225 "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I') and skymap='hsc_rings_v1'", 

226 visits, zpOffsets, 125, 64) 

227 

228 def test_fgcmcalTractPipeline(self): 

229 # Set numpy seed for stability 

230 np.random.seed(seed=1000) 

231 

232 instName = 'HSC' 

233 testName = 'testfgcmcaltract' 

234 

235 nBand = 3 

236 i0Std = np.array(I0STD) 

237 i10Std = np.array(I10STD) 

238 i0Recon = np.array(I0RECON) 

239 i10Recon = np.array(I10RECON) 

240 

241 self._testFgcmMakeLut(instName, testName, 

242 nBand, i0Std, i0Recon, i10Std, i10Recon) 

243 

244 rawRepeatability = np.array([0.0, 

245 0.01201036876093324, 

246 0.003979326480379284]) 

247 filterNCalibMap = {'HSC-R': 48, 

248 'HSC-I': 75} 

249 

250 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

251 tract = 9697 

252 

253 self._testFgcmCalibrateTract(instName, testName, 

254 visits, tract, 'hsc_rings_v1', 

255 rawRepeatability, filterNCalibMap) 

256 

257 

258class TestMemory(lsst.utils.tests.MemoryTestCase): 

259 pass 

260 

261 

262def setup_module(module): 

263 lsst.utils.tests.init() 

264 

265 

266if __name__ == "__main__": 266 ↛ 267line 266 didn't jump to line 267 because the condition on line 266 was never true

267 lsst.utils.tests.init() 

268 unittest.main()