Coverage for tests / test_fgcmcal_hsc.py: 23%

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Test the fgcmcal code with testdata_jointcal/hsc. 

24 

25Run test suite on fgcmcal using Gen3 HSC data from testdata_jointcal. 

26""" 

27import unittest 

28import os 

29import tempfile 

30import numpy as np 

31 

32# Need to import pyproj to prevent file handle leakage since importing 

33# pyproj automatically opens proj.db and never closes it. We can not wait 

34# for some dependent code to import it whilst the test is running since then 

35# the leak checker will think it is a leak. 

36# TODO: Remove import after completing DM-54643. Use DM-54656 

37try: 

38 import pyproj # noqa: F401 

39except ImportError: 

40 pass 

41 

42# Ensure that matplotlib doesn't try to open a display during testing. 

43import matplotlib 

44matplotlib.use("Agg") 

45 

46import lsst.utils # noqa: E402 

47import lsst.pipe.tasks # noqa: E402 

48import lsst.daf.butler.cli.cliLog # noqa: E402 

49 

50import fgcmcalTestBase # noqa: E402 

51 

52 

53ROOT = os.path.abspath(os.path.dirname(__file__)) 

54 

55I0STD = [0.08294534, 0.07877351, 0.06464688] 

56I10STD = [-0.000091981, -0.00061516, -0.00063434] 

57I0RECON = [0.07322179342588758, 0.0689530429, 0.05600673] 

58I10RECON = [-4.490243571049125, -7.01786443508, 3.62738180611] 

59 

60 

61class FgcmcalTestHSC(fgcmcalTestBase.FgcmcalTestBase, lsst.utils.tests.TestCase): 

62 @classmethod 

63 def setUpClass(cls): 

64 try: 

65 cls.dataDir = lsst.utils.getPackageDir('testdata_jointcal') 

66 except LookupError: 

67 raise unittest.SkipTest("testdata_jointcal not setup") 

68 try: 

69 lsst.utils.getPackageDir('obs_subaru') 

70 except LookupError: 

71 raise unittest.SkipTest("obs_subaru not setup") 

72 

73 lsst.daf.butler.cli.cliLog.CliLog.initLog(longlog=False) 

74 

75 cls.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestFgcm-") 

76 

77 cls._importRepository('lsst.obs.subaru.HyperSuprimeCam', 

78 os.path.join(cls.dataDir, 'hsc/repo'), 

79 os.path.join(cls.dataDir, 'hsc', 'exports.yaml')) 

80 

81 def test_fgcmcalPipelineBuildFromTable(self): 

82 """Test running the full pipeline, using older association code. 

83 

84 This test uses the FgcmBuildStarsFromTableTask instead of the new 

85 FgcmBuildFromIsolatedStarsTask. 

86 """ 

87 # Set numpy seed for stability 

88 np.random.seed(seed=1000) 

89 

90 instName = 'HSC' 

91 testName = 'testfgcmcalpipe' 

92 

93 nBand = 3 

94 i0Std = np.array(I0STD) 

95 i10Std = np.array(I10STD) 

96 i0Recon = np.array(I0RECON) 

97 i10Recon = np.array(I10RECON) 

98 

99 self._testFgcmMakeLut(instName, testName, 

100 nBand, i0Std, i0Recon, i10Std, i10Recon) 

101 

102 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

103 

104 nStar = 305 

105 nObs = 3789 

106 

107 self._testFgcmBuildStarsTable(instName, testName, 

108 "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I')", 

109 visits, nStar, nObs) 

110 

111 nZp = 1120 

112 nGoodZp = 123 

113 nOkZp = 123 

114 nBadZp = 997 

115 nStdStars = 237 

116 nPlots = 66 

117 

118 # We need an extra config file to turn off parquet format. 

119 extraConfigFile = os.path.join(self.testDir, "turn_off_parquet.py") 

120 with open(extraConfigFile, "w") as f: 

121 f.write("config.useParquetCatalogFormat = False\n") 

122 

123 self._testFgcmFitCycle(instName, testName, 

124 0, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True, 

125 extraConfig=extraConfigFile) 

126 self._testFgcmFitCycle(instName, testName, 

127 1, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True, 

128 extraConfig=extraConfigFile) 

129 

130 # We need to create an extra config file to turn on "sub-ccd gray" for testing. 

131 # We also want to exercise the code path setting useExposureReferenceOffset = False. 

132 extraConfigFile = os.path.join(self.testDir, "cycle03_patch_config.py") 

133 with open(extraConfigFile, "w") as f: 

134 f.write("config.useParquetCatalogFormat = False\n") 

135 f.write("config.isFinalCycle = True\n") 

136 f.write("config.ccdGraySubCcdDict = {'g': True, 'r': True, 'i': True}\n") 

137 f.write("config.useExposureReferenceOffset = False") 

138 

139 self._testFgcmFitCycle(instName, testName, 

140 2, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, 

141 extraConfig=extraConfigFile) 

142 

143 def test_fgcmcalPipeline(self): 

144 """Test running the full pipeline, using new isolated star association code. 

145 

146 This test uses the FgcmBuildFromIsolatedStarsTask instead of the old 

147 FgcmBuildStarsFromTableTask. 

148 """ 

149 # Set numpy seed for stability 

150 np.random.seed(seed=1000) 

151 

152 instName = 'HSC' 

153 testName = 'testfgcmcalpipe' 

154 

155 nBand = 3 

156 i0Std = np.array(I0STD) 

157 i10Std = np.array(I10STD) 

158 i0Recon = np.array(I0RECON) 

159 i10Recon = np.array(I10RECON) 

160 

161 self._testFgcmMakeLut(instName, testName, 

162 nBand, i0Std, i0Recon, i10Std, i10Recon) 

163 

164 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

165 

166 nStar = 292 

167 nObs = 1790 

168 

169 self._testFgcmBuildFromIsolatedStars( 

170 instName, 

171 testName, 

172 "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I')", 

173 visits, 

174 nStar, 

175 nObs, 

176 ) 

177 

178 nZp = 1120 

179 nGoodZp = 123 

180 nOkZp = 123 

181 nBadZp = 997 

182 nStdStars = 222 

183 nPlots = 66 

184 

185 self._testFgcmFitCycle(instName, testName, 

186 0, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True) 

187 self._testFgcmFitCycle(instName, testName, 

188 1, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True) 

189 

190 # We need to create an extra config file to turn on "sub-ccd gray" for testing. 

191 # We also want to exercise the code path setting useExposureReferenceOffset = False. 

192 extraConfigFile = os.path.join(self.testDir, "cycle03_patch_config.py") 

193 with open(extraConfigFile, "w") as f: 

194 f.write("config.isFinalCycle = True\n") 

195 f.write("config.ccdGraySubCcdDict = {'g': True, 'r': True, 'i': True}\n") 

196 f.write("config.useExposureReferenceOffset = False") 

197 

198 self._testFgcmFitCycle(instName, testName, 

199 2, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, 

200 extraConfig=extraConfigFile) 

201 

202 zpOffsets = np.array([-0.000842530163936317, 

203 0.0047900681383907795]) 

204 

205 self._testFgcmOutputProducts(instName, testName, 

206 zpOffsets, 36236, 87, 'i', 1, 'hsc_rings_v1') 

207 

208 # Test a single detector illumination correction. 

209 self._testFgcmOutputIlluminationCorrection(instName, testName, 51) 

210 

211 def test_fgcmcalMultipleFitPipeline(self): 

212 # Set numpy seed for stability 

213 np.random.seed(seed=1000) 

214 

215 instName = 'HSC' 

216 testName = 'testfgcmcalmultiple' 

217 

218 nBand = 3 

219 i0Std = np.array(I0STD) 

220 i10Std = np.array(I10STD) 

221 i0Recon = np.array(I0RECON) 

222 i10Recon = np.array(I10RECON) 

223 

224 self._testFgcmMakeLut(instName, testName, 

225 nBand, i0Std, i0Recon, i10Std, i10Recon) 

226 

227 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

228 

229 # These are slightly different from above due to the configuration change 

230 # mid-way in the separate fits. 

231 zpOffsets = np.array([0.0011443052208051085, 

232 0.0015734810149297118]) 

233 

234 self._testFgcmMultiFit(instName, testName, 

235 "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I') and skymap='hsc_rings_v1'", 

236 visits, zpOffsets, 125, 64) 

237 

238 def test_fgcmcalTractPipeline(self): 

239 # Set numpy seed for stability 

240 np.random.seed(seed=1000) 

241 

242 instName = 'HSC' 

243 testName = 'testfgcmcaltract' 

244 

245 nBand = 3 

246 i0Std = np.array(I0STD) 

247 i10Std = np.array(I10STD) 

248 i0Recon = np.array(I0RECON) 

249 i10Recon = np.array(I10RECON) 

250 

251 self._testFgcmMakeLut(instName, testName, 

252 nBand, i0Std, i0Recon, i10Std, i10Recon) 

253 

254 rawRepeatability = np.array([0.0, 

255 0.01201036876093324, 

256 0.003979326480379284]) 

257 filterNCalibMap = {'HSC-R': 48, 

258 'HSC-I': 75} 

259 

260 visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236] 

261 tract = 9697 

262 

263 self._testFgcmCalibrateTract(instName, testName, 

264 visits, tract, 'hsc_rings_v1', 

265 rawRepeatability, filterNCalibMap) 

266 

267 

268class TestMemory(lsst.utils.tests.MemoryTestCase): 

269 pass 

270 

271 

272def setup_module(module): 

273 lsst.utils.tests.init() 

274 

275 

276if __name__ == "__main__": 276 ↛ 277line 276 didn't jump to line 277 because the condition on line 276 was never true

277 lsst.utils.tests.init() 

278 unittest.main()