Coverage for tests/test_jointcal_cfht.py: 20%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

167 statements  

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23import os 

24import tempfile 

25 

26from astropy import units as u 

27 

28import lsst.geom 

29import lsst.utils 

30import lsst.pex.exceptions 

31import lsst.pex.config 

32 

33import jointcalTestBase 

34 

35 

36# for MemoryTestCase 

37def setup_module(module): 

38 lsst.utils.tests.init() 

39 

40 

41class JointcalTestCFHT(jointcalTestBase.JointcalTestBase, lsst.utils.tests.TestCase): 

42 

43 @classmethod 

44 def setUpClass(cls): 

45 try: 

46 cls.data_dir = lsst.utils.getPackageDir('testdata_jointcal') 

47 except LookupError: 

48 raise unittest.SkipTest("testdata_jointcal not setup") 

49 try: 

50 lsst.utils.getPackageDir('obs_cfht') 

51 except LookupError: 

52 raise unittest.SkipTest("obs_cfht not setup") 

53 

54 def setUp(self): 

55 # NOTE: refcat-comparison RMS error is worse now, because the 

56 # comparison code is not applying the proper motion data. 

57 # See Readme for an explanation of this empirical value. 

58 self.dist_rms_absolute = 70e-3*u.arcsecond 

59 

60 do_plot = False 

61 

62 # center of the cfht validation_data catalog 

63 center = lsst.geom.SpherePoint(214.884832, 52.6622199, lsst.geom.degrees) 

64 radius = 3*lsst.geom.degrees 

65 

66 input_dir = os.path.join(self.data_dir, 'cfht') 

67 all_visits = [849375, 850587] 

68 

69 self.setUp_base(center, radius, 

70 input_dir=input_dir, 

71 all_visits=all_visits, 

72 do_plot=do_plot, 

73 log_level="DEBUG") 

74 

75 def test_jointcalTask_2_visits(self): 

76 """Test the simple models with two visits and check that some debug 

77 output files also get created. 

78 """ 

79 self.config = lsst.jointcal.jointcal.JointcalConfig() 

80 self.config.astrometryModel = "simple" 

81 self.config.photometryModel = "simpleFlux" 

82 self.config.writeInitialModel = True # write the initial models 

83 # to test whether we got the expected chi2 contribution files. 

84 self.config.writeChi2FilesInitialFinal = True 

85 # use a temporary directory for debug output, to prevent test collisions 

86 with tempfile.TemporaryDirectory() as tempdir: 

87 self.config.debugOutputPath = tempdir 

88 

89 # See Readme for an explanation of these empirical values. 

90 dist_rms_relative = 16e-3*u.arcsecond 

91 pa1 = 0.049 

92 metrics = {'collected_astrometry_refStars': 867, 

93 'collected_photometry_refStars': 11570, 

94 'selected_astrometry_refStars': 332, 

95 'selected_photometry_refStars': 2225, 

96 'associated_astrometry_fittedStars': 2272, 

97 'associated_photometry_fittedStars': 2272, 

98 'selected_astrometry_fittedStars': 1229, 

99 'selected_photometry_fittedStars': 2232, 

100 'selected_astrometry_ccdImages': 12, 

101 'selected_photometry_ccdImages': 12, 

102 'astrometry_final_chi2': 1145.457, 

103 'astrometry_final_ndof': 1868, 

104 'photometry_final_chi2': 11624.3, 

105 'photometry_final_ndof': 2778 

106 } 

107 

108 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, pa1, metrics=metrics) 

109 

110 # Check for the existence of the chi2 contribution files. 

111 expected = ['photometry_initial_chi2-0_r.MP9601', 'astrometry_initial_chi2-0_r.MP9601', 

112 'photometry_final_chi2-0_r.MP9601', 'astrometry_final_chi2-0_r.MP9601'] 

113 for partial in expected: 

114 name = os.path.join(tempdir, partial+'-ref.csv') 

115 self.assertTrue(os.path.exists(name), msg="Did not find file %s"%name) 

116 name = os.path.join(tempdir, partial+'-meas.csv') 

117 self.assertTrue(os.path.exists(name), msg='Did not find file %s'%name) 

118 

119 expected = ["initial_astrometry_model-0_r.MP9601.txt", "initial_photometry_model-0_r.MP9601.txt"] 

120 for name in expected: 

121 fullpath = os.path.join(tempdir, name) 

122 self.assertTrue(os.path.exists(fullpath), msg=f"Did not find file {fullpath}") 

123 

124 def setup_jointcalTask_2_visits_constrainedAstrometry(self): 

125 """Set default values for the constrainedAstrometry tests, and make 

126 the differences between each test and the defaults more obvious. 

127 """ 

128 self.config = lsst.jointcal.jointcal.JointcalConfig() 

129 self.config.astrometryModel = "constrained" 

130 self.config.doPhotometry = False 

131 self.jointcalStatistics.do_photometry = False 

132 

133 # See Readme for an explanation of these empirical values. 

134 dist_rms_relative = 16e-3*u.arcsecond 

135 metrics = {'collected_astrometry_refStars': 867, 

136 'selected_astrometry_refStars': 332, 

137 'associated_astrometry_fittedStars': 2272, 

138 'selected_astrometry_fittedStars': 1229, 

139 'selected_astrometry_ccdImages': 12, 

140 'astrometry_final_chi2': 1100.75, 

141 'astrometry_final_ndof': 1910, 

142 } 

143 

144 return dist_rms_relative, metrics 

145 

146 def test_jointcalTask_2_visits_constrainedAstrometry_no_photometry(self): 

147 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry() 

148 self.config.writeInitialModel = True # write the initial models 

149 # use a temporary directory for debug output, to prevent test collisions 

150 with tempfile.TemporaryDirectory() as tempdir: 

151 self.config.debugOutputPath = tempdir 

152 

153 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics) 

154 filename = os.path.join(tempdir, "initial_astrometry_model-0_r.MP9601.txt") 

155 self.assertTrue(os.path.exists(filename), msg=f"Did not find file {filename}") 

156 

157 def test_jointcalTask_2_visits_constrainedAstrometry_no_rank_update(self): 

158 """Demonstrate that skipping the rank update doesn't substantially affect astrometry. 

159 """ 

160 relative_error, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry() 

161 metrics['astrometry_final_chi2'] = 1069.538 

162 metrics['astrometry_final_ndof'] = 1644 

163 

164 self.config.astrometryDoRankUpdate = False 

165 

166 self._testJointcalTask(2, relative_error, self.dist_rms_absolute, None, metrics=metrics) 

167 

168 def test_jointcalTask_2_visits_constrainedAstrometry_4sigma_outliers(self): 

169 """4 sigma outlier rejection means fewer available sources after the 

170 fitter converges, resulting in a smaller ndof and chi2. 

171 """ 

172 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry() 

173 self.config.outlierRejectSigma = 4 

174 metrics['astrometry_final_chi2'] = 800.919 

175 metrics['astrometry_final_ndof'] = 1796 

176 

177 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics) 

178 

179 def test_jointcalTask_2_visits_constrainedAstrometry_astrometryOutlierRelativeTolerance(self): 

180 """Test that astrometryOutlierRelativeTolerance changes the fit. Setting 

181 1% for the astrometryOutlierRelativeTolerance will result in higher chi2 

182 and ndof. 

183 """ 

184 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry() 

185 self.config.astrometryOutlierRelativeTolerance = 0.01 

186 metrics['astrometry_final_chi2'] = 1393.40 

187 metrics['astrometry_final_ndof'] = 1982 

188 

189 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics) 

190 

191 def test_jointcalTask_2_visits_constrainedAstrometry_astrometryReferenceUncertainty_smaller(self): 

192 """Test with a smaller fake reference uncertainty: chi2 will be higher.""" 

193 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry() 

194 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 

195 'tests/config/astrometryReferenceErr-config.py') 

196 self.configfiles.append(test_config) 

197 metrics['astrometry_final_chi2'] = 1303.55 

198 metrics['astrometry_final_ndof'] = 2068 

199 

200 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics) 

201 

202 def test_jointcalTask_2_visits_constrainedAstrometry_astrometryReferenceUncertainty_None_fails(self): 

203 """The default `None` should fail for the existing refcats that have no position errors.""" 

204 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry() 

205 # This is the default, but we override it in tests/config/config.py, 

206 # because none of the existing test refcats have errors. So we have to 

207 # re-override it here. 

208 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 

209 'tests/config/astrometryReferenceErr-None-config.py') 

210 self.configfiles.append(test_config) 

211 with self.assertRaises(lsst.pex.config.FieldValidationError): 

212 self._testJointcalTask(2, None, None, None) 

213 

214 def setup_jointcalTask_2_visits_constrainedPhotometry(self): 

215 """Set default values for the constrainedPhotometry tests, and make 

216 the differences between each test and the defaults more obvious. 

217 """ 

218 self.config = lsst.jointcal.jointcal.JointcalConfig() 

219 self.config.photometryModel = "constrainedFlux" 

220 self.config.doAstrometry = False 

221 self.jointcalStatistics.do_astrometry = False 

222 

223 # See Readme for an explanation of these empirical values. 

224 pa1 = 0.09 

225 metrics = {'collected_photometry_refStars': 11570, 

226 'selected_photometry_refStars': 2225, 

227 'associated_photometry_fittedStars': 2272, 

228 'selected_photometry_fittedStars': 2232, 

229 'selected_photometry_ccdImages': 12, 

230 'photometry_final_chi2': 11649.7, 

231 'photometry_final_ndof': 2729 

232 } 

233 return pa1, metrics 

234 

235 def test_jointcalTask_2_visits_constrainedPhotometry_no_astrometry(self): 

236 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

237 self.config.writeInitialModel = True # write the initial models 

238 # use a temporary directory for debug output, to prevent test collisions 

239 with tempfile.TemporaryDirectory() as tempdir: 

240 self.config.debugOutputPath = tempdir 

241 

242 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

243 filename = os.path.join(tempdir, "initial_photometry_model-0_r.MP9601.txt") 

244 self.assertTrue(os.path.exists(filename), msg=f"Did not find file {filename}") 

245 

246 def test_jointcalTask_2_visits_constrainedPhotometry_no_rank_update(self): 

247 """Demonstrate that skipping the rank update doesn't substantially affect photometry. 

248 """ 

249 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

250 self.config.photometryDoRankUpdate = False 

251 

252 # The constrainedPhotometry model is not purely linear, so a small 

253 # change in final chi2 is possible. 

254 metrics['photometry_final_chi2'] = 11396.27 

255 metrics['photometry_final_ndof'] = 2695 

256 

257 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

258 

259 def test_jointcalTask_2_visits_constrainedPhotometry_lineSearch(self): 

260 """Activating the line search should only slightly change the chi2. 

261 """ 

262 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

263 self.config.allowLineSearch = True 

264 

265 # Activating line search for constrainedPhotometry should result in 

266 # nearly the same final fit (the system is somewhat non-linear, so it 

267 # may not be exactly the same: check the "Line search scale factor" 

268 # lines in the DEBUG log for values that are not ~1 for proof). 

269 pa1 = 0.14 

270 metrics['photometry_final_chi2'] = 10500.4 

271 metrics['photometry_final_ndof'] = 2712 

272 

273 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

274 

275 def test_jointcalTask_2_visits_constrainedPhotometry_flagged(self): 

276 """Test the use of the FlaggedSourceSelector.""" 

277 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

278 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 

279 'tests/config/cfht-flagged-config.py') 

280 self.configfiles.append(test_config) 

281 # Reduce warnings due to flaggedSourceSelector having fewer sources than astrometrySourceSelector. 

282 self.config.minMeasuredStarsPerCcd = 30 

283 self.config.minRefStarsPerCcd = 20 

284 

285 pa1 = 0.026 

286 metrics['selected_photometry_refStars'] = 265 

287 metrics['associated_photometry_fittedStars'] = 265 

288 metrics['selected_photometry_fittedStars'] = 265 

289 metrics['photometry_final_chi2'] = 392.816 

290 metrics['photometry_final_ndof'] = 294 

291 

292 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

293 

294 def test_jointcalTask_2_visits_constrainedMagnitude_no_astrometry(self): 

295 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

296 self.config.photometryModel = "constrainedMagnitude" 

297 

298 # The resulting fit should be close to the constrainedFlux model: 

299 # there are few CCDs and 2 visits, so there's not a lot of complexity 

300 # in this case to distinguish the flux vs. magnitude models. 

301 metrics['photometry_final_chi2'] = 9455.06 

302 metrics['photometry_final_ndof'] = 2710 

303 

304 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

305 

306 def test_jointcalTask_2_visits_constrainedFlux_pedestal(self): 

307 """Test that forcing a systematic flux error results in a lower chi2. 

308 """ 

309 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

310 # median fluxErr/flux in ps1 is 0.11, so we have to make this bigger 

311 # than that to actually allow more slop in the fit. 

312 self.config.photometryErrorPedestal = 0.2 

313 

314 # We're allowing more error in the fit, so PA1 may be worse. 

315 pa1 = 0.21 

316 # Final chi2 is much lower, because all sources contribute more error. 

317 metrics['photometry_final_chi2'] = 3214.78 

318 # ndof may change; slightly different likelihood contours, and fewer 

319 # reference sources rejected. 

320 metrics['photometry_final_ndof'] = 3154 

321 

322 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

323 

324 def test_jointcalTask_2_visits_constrainedMagnitude_pedestal(self): 

325 """Test that forcing a systematic flux error results in a lower chi2. 

326 """ 

327 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry() 

328 self.config.photometryModel = "constrainedMagnitude" 

329 # median fluxErr/flux in ps1 is 0.11, so we have to make this bigger 

330 # than that to actually allow more slop in the fit. 

331 self.config.photometryErrorPedestal = 0.2 

332 

333 # We're allowing more error in the fit, so PA1 may be worse. 

334 pa1 = 0.19 

335 # Final chi2 is much lower, because all sources contribute more error. 

336 metrics['photometry_final_chi2'] = 3092.39 

337 # ndof may change; slightly different likelihood contours, and fewer 

338 # reference sources rejected. 

339 metrics['photometry_final_ndof'] = 3129 

340 

341 self._testJointcalTask(2, None, None, pa1, metrics=metrics) 

342 

343 

344class MemoryTester(lsst.utils.tests.MemoryTestCase): 

345 pass 

346 

347 

348if __name__ == "__main__": 348 ↛ 349line 348 didn't jump to line 349, because the condition on line 348 was never true

349 lsst.utils.tests.init() 

350 unittest.main()