Coverage for tests/test_jointcal_cfht.py: 20%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
23import os
24import tempfile
26from astropy import units as u
28import lsst.geom
29import lsst.utils
30import lsst.pex.exceptions
31import lsst.pex.config
33import jointcalTestBase
36# for MemoryTestCase
37def setup_module(module):
38 lsst.utils.tests.init()
41class JointcalTestCFHT(jointcalTestBase.JointcalTestBase, lsst.utils.tests.TestCase):
43 @classmethod
44 def setUpClass(cls):
45 try:
46 cls.data_dir = lsst.utils.getPackageDir('testdata_jointcal')
47 except LookupError:
48 raise unittest.SkipTest("testdata_jointcal not setup")
49 try:
50 lsst.utils.getPackageDir('obs_cfht')
51 except LookupError:
52 raise unittest.SkipTest("obs_cfht not setup")
54 def setUp(self):
55 # NOTE: refcat-comparison RMS error is worse now, because the
56 # comparison code is not applying the proper motion data.
57 # See Readme for an explanation of this empirical value.
58 self.dist_rms_absolute = 70e-3*u.arcsecond
60 do_plot = False
62 # center of the cfht validation_data catalog
63 center = lsst.geom.SpherePoint(214.884832, 52.6622199, lsst.geom.degrees)
64 radius = 3*lsst.geom.degrees
66 input_dir = os.path.join(self.data_dir, 'cfht')
67 all_visits = [849375, 850587]
69 self.setUp_base(center, radius,
70 input_dir=input_dir,
71 all_visits=all_visits,
72 do_plot=do_plot,
73 log_level="DEBUG")
75 def test_jointcalTask_2_visits(self):
76 """Test the simple models with two visits and check that some debug
77 output files also get created.
78 """
79 self.config = lsst.jointcal.jointcal.JointcalConfig()
80 self.config.astrometryModel = "simple"
81 self.config.photometryModel = "simpleFlux"
82 self.config.writeInitialModel = True # write the initial models
83 # to test whether we got the expected chi2 contribution files.
84 self.config.writeChi2FilesInitialFinal = True
85 # use a temporary directory for debug output, to prevent test collisions
86 with tempfile.TemporaryDirectory() as tempdir:
87 self.config.debugOutputPath = tempdir
89 # See Readme for an explanation of these empirical values.
90 dist_rms_relative = 16e-3*u.arcsecond
91 pa1 = 0.049
92 metrics = {'astrometry_collected_refStars': 867,
93 'photometry_collected_refStars': 11570,
94 'astrometry_prepared_refStars': 332,
95 'photometry_prepared_refStars': 2225,
96 'astrometry_matched_fittedStars': 2272,
97 'photometry_matched_fittedStars': 2272,
98 'astrometry_prepared_fittedStars': 1229,
99 'photometry_prepared_fittedStars': 2232,
100 'astrometry_prepared_ccdImages': 12,
101 'photometry_prepared_ccdImages': 12,
102 'astrometry_final_chi2': 1145.457,
103 'astrometry_final_ndof': 1868,
104 'photometry_final_chi2': 11624.3,
105 'photometry_final_ndof': 2778
106 }
108 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, pa1, metrics=metrics)
110 # Check for the existence of the chi2 contribution files.
111 expected = ['photometry_initial_chi2-0_r.MP9601', 'astrometry_initial_chi2-0_r.MP9601',
112 'photometry_final_chi2-0_r.MP9601', 'astrometry_final_chi2-0_r.MP9601']
113 for partial in expected:
114 name = os.path.join(tempdir, partial+'-ref.csv')
115 self.assertTrue(os.path.exists(name), msg="Did not find file %s"%name)
116 name = os.path.join(tempdir, partial+'-meas.csv')
117 self.assertTrue(os.path.exists(name), msg='Did not find file %s'%name)
119 expected = ["initial_astrometry_model-0_r.MP9601.txt", "initial_photometry_model-0_r.MP9601.txt"]
120 for name in expected:
121 fullpath = os.path.join(tempdir, name)
122 self.assertTrue(os.path.exists(fullpath), msg=f"Did not find file {fullpath}")
124 def setup_jointcalTask_2_visits_constrainedAstrometry(self):
125 """Set default values for the constrainedAstrometry tests, and make
126 the differences between each test and the defaults more obvious.
127 """
128 self.config = lsst.jointcal.jointcal.JointcalConfig()
129 self.config.astrometryModel = "constrained"
130 self.config.doPhotometry = False
131 self.jointcalStatistics.do_photometry = False
133 # See Readme for an explanation of these empirical values.
134 dist_rms_relative = 16e-3*u.arcsecond
135 metrics = {'astrometry_collected_refStars': 867,
136 'astrometry_prepared_refStars': 332,
137 'astrometry_matched_fittedStars': 2272,
138 'astrometry_prepared_fittedStars': 1229,
139 'astrometry_prepared_ccdImages': 12,
140 'astrometry_final_chi2': 1100.75,
141 'astrometry_final_ndof': 1910,
142 }
144 return dist_rms_relative, metrics
146 def test_jointcalTask_2_visits_constrainedAstrometry_no_photometry(self):
147 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry()
148 self.config.writeInitialModel = True # write the initial models
149 # use a temporary directory for debug output, to prevent test collisions
150 with tempfile.TemporaryDirectory() as tempdir:
151 self.config.debugOutputPath = tempdir
153 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics)
154 filename = os.path.join(tempdir, "initial_astrometry_model-0_r.MP9601.txt")
155 self.assertTrue(os.path.exists(filename), msg=f"Did not find file {filename}")
157 def test_jointcalTask_2_visits_constrainedAstrometry_no_rank_update(self):
158 """Demonstrate that skipping the rank update doesn't substantially affect astrometry.
159 """
160 relative_error, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry()
161 metrics['astrometry_final_chi2'] = 1069.538
162 metrics['astrometry_final_ndof'] = 1644
164 self.config.astrometryDoRankUpdate = False
166 self._testJointcalTask(2, relative_error, self.dist_rms_absolute, None, metrics=metrics)
168 def test_jointcalTask_2_visits_constrainedAstrometry_4sigma_outliers(self):
169 """4 sigma outlier rejection means fewer available sources after the
170 fitter converges, resulting in a smaller ndof and chi2.
171 """
172 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry()
173 self.config.outlierRejectSigma = 4
174 metrics['astrometry_final_chi2'] = 800.919
175 metrics['astrometry_final_ndof'] = 1796
177 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics)
179 def test_jointcalTask_2_visits_constrainedAstrometry_astrometryOutlierRelativeTolerance(self):
180 """Test that astrometryOutlierRelativeTolerance changes the fit. Setting
181 1% for the astrometryOutlierRelativeTolerance will result in higher chi2
182 and ndof.
183 """
184 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry()
185 self.config.astrometryOutlierRelativeTolerance = 0.01
186 metrics['astrometry_final_chi2'] = 1393.40
187 metrics['astrometry_final_ndof'] = 1982
189 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics)
191 def test_jointcalTask_2_visits_constrainedAstrometry_astrometryReferenceUncertainty_smaller(self):
192 """Test with a smaller fake reference uncertainty: chi2 will be higher."""
193 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry()
194 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'),
195 'tests/config/astrometryReferenceErr-config.py')
196 self.configfiles.append(test_config)
197 metrics['astrometry_final_chi2'] = 1303.55
198 metrics['astrometry_final_ndof'] = 2068
200 self._testJointcalTask(2, dist_rms_relative, self.dist_rms_absolute, None, metrics=metrics)
202 def test_jointcalTask_2_visits_constrainedAstrometry_astrometryReferenceUncertainty_None_fails(self):
203 """The default `None` should fail for the existing refcats that have no position errors."""
204 dist_rms_relative, metrics = self.setup_jointcalTask_2_visits_constrainedAstrometry()
205 # This is the default, but we override it in tests/config/config.py,
206 # because none of the existing test refcats have errors. So we have to
207 # re-override it here.
208 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'),
209 'tests/config/astrometryReferenceErr-None-config.py')
210 self.configfiles.append(test_config)
211 with self.assertRaises(lsst.pex.config.FieldValidationError):
212 self._testJointcalTask(2, None, None, None)
214 def setup_jointcalTask_2_visits_constrainedPhotometry(self):
215 """Set default values for the constrainedPhotometry tests, and make
216 the differences between each test and the defaults more obvious.
217 """
218 self.config = lsst.jointcal.jointcal.JointcalConfig()
219 self.config.photometryModel = "constrainedFlux"
220 self.config.doAstrometry = False
221 self.jointcalStatistics.do_astrometry = False
223 # See Readme for an explanation of these empirical values.
224 pa1 = 0.09
225 metrics = {'photometry_collected_refStars': 11570,
226 'photometry_prepared_refStars': 2225,
227 'photometry_matched_fittedStars': 2272,
228 'photometry_prepared_fittedStars': 2232,
229 'photometry_prepared_ccdImages': 12,
230 'photometry_final_chi2': 11649.7,
231 'photometry_final_ndof': 2729
232 }
233 return pa1, metrics
235 def test_jointcalTask_2_visits_constrainedPhotometry_no_astrometry(self):
236 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
237 self.config.writeInitialModel = True # write the initial models
238 # use a temporary directory for debug output, to prevent test collisions
239 with tempfile.TemporaryDirectory() as tempdir:
240 self.config.debugOutputPath = tempdir
242 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
243 filename = os.path.join(tempdir, "initial_photometry_model-0_r.MP9601.txt")
244 self.assertTrue(os.path.exists(filename), msg=f"Did not find file {filename}")
246 def test_jointcalTask_2_visits_constrainedPhotometry_no_rank_update(self):
247 """Demonstrate that skipping the rank update doesn't substantially affect photometry.
248 """
249 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
250 self.config.photometryDoRankUpdate = False
252 # The constrainedPhotometry model is not purely linear, so a small
253 # change in final chi2 is possible.
254 metrics['photometry_final_chi2'] = 11396.27
255 metrics['photometry_final_ndof'] = 2695
257 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
259 def test_jointcalTask_2_visits_constrainedPhotometry_lineSearch(self):
260 """Activating the line search should only slightly change the chi2.
261 """
262 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
263 self.config.allowLineSearch = True
265 # Activating line search for constrainedPhotometry should result in
266 # nearly the same final fit (the system is somewhat non-linear, so it
267 # may not be exactly the same: check the "Line search scale factor"
268 # lines in the DEBUG log for values that are not ~1 for proof).
269 pa1 = 0.14
270 metrics['photometry_final_chi2'] = 10500.4
271 metrics['photometry_final_ndof'] = 2712
273 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
275 def test_jointcalTask_2_visits_constrainedPhotometry_flagged(self):
276 """Test the use of the FlaggedSourceSelector."""
277 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
278 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'),
279 'tests/config/cfht-flagged-config.py')
280 self.configfiles.append(test_config)
281 # Reduce warnings due to flaggedSourceSelector having fewer sources than astrometrySourceSelector.
282 self.config.minMeasuredStarsPerCcd = 30
283 self.config.minRefStarsPerCcd = 20
285 pa1 = 0.026
286 metrics['photometry_prepared_refStars'] = 265
287 metrics['photometry_matched_fittedStars'] = 265
288 metrics['photometry_prepared_fittedStars'] = 265
289 metrics['photometry_final_chi2'] = 392.816
290 metrics['photometry_final_ndof'] = 294
292 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
294 def test_jointcalTask_2_visits_constrainedMagnitude_no_astrometry(self):
295 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
296 self.config.photometryModel = "constrainedMagnitude"
298 # The resulting fit should be close to the constrainedFlux model:
299 # there are few CCDs and 2 visits, so there's not a lot of complexity
300 # in this case to distinguish the flux vs. magnitude models.
301 metrics['photometry_final_chi2'] = 9455.06
302 metrics['photometry_final_ndof'] = 2710
304 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
306 def test_jointcalTask_2_visits_constrainedFlux_pedestal(self):
307 """Test that forcing a systematic flux error results in a lower chi2.
308 """
309 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
310 # median fluxErr/flux in ps1 is 0.11, so we have to make this bigger
311 # than that to actually allow more slop in the fit.
312 self.config.photometryErrorPedestal = 0.2
314 # We're allowing more error in the fit, so PA1 may be worse.
315 pa1 = 0.21
316 # Final chi2 is much lower, because all sources contribute more error.
317 metrics['photometry_final_chi2'] = 3214.78
318 # ndof may change; slightly different likelihood contours, and fewer
319 # reference sources rejected.
320 metrics['photometry_final_ndof'] = 3154
322 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
324 def test_jointcalTask_2_visits_constrainedMagnitude_pedestal(self):
325 """Test that forcing a systematic flux error results in a lower chi2.
326 """
327 pa1, metrics = self.setup_jointcalTask_2_visits_constrainedPhotometry()
328 self.config.photometryModel = "constrainedMagnitude"
329 # median fluxErr/flux in ps1 is 0.11, so we have to make this bigger
330 # than that to actually allow more slop in the fit.
331 self.config.photometryErrorPedestal = 0.2
333 # We're allowing more error in the fit, so PA1 may be worse.
334 pa1 = 0.19
335 # Final chi2 is much lower, because all sources contribute more error.
336 metrics['photometry_final_chi2'] = 3092.39
337 # ndof may change; slightly different likelihood contours, and fewer
338 # reference sources rejected.
339 metrics['photometry_final_ndof'] = 3129
341 self._testJointcalTask(2, None, None, pa1, metrics=metrics)
344class MemoryTester(lsst.utils.tests.MemoryTestCase):
345 pass
348if __name__ == "__main__": 348 ↛ 349line 348 didn't jump to line 349, because the condition on line 348 was never true
349 lsst.utils.tests.init()
350 unittest.main()