Coverage for tests/test_linearity.py: 14%
187 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-04 09:08 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-04 09:08 +0000
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe linearity code."""
27import unittest
28import numpy as np
30import lsst.utils
31import lsst.utils.tests
33from lsst.ip.isr import PhotonTransferCurveDataset
35import lsst.afw.image
36import lsst.afw.math
37from lsst.cp.pipe import LinearitySolveTask
38from lsst.cp.pipe.ptc import PhotonTransferCurveSolveTask
39from lsst.cp.pipe.utils import funcPolynomial
40from lsst.ip.isr.isrMock import FlatMock, IsrMock
43class FakeCamera(list):
44 def getName(self):
45 return "FakeCam"
48class LinearityTaskTestCase(lsst.utils.tests.TestCase):
49 """Test case for the linearity tasks."""
51 def setUp(self):
52 mock_image_config = IsrMock.ConfigClass()
53 mock_image_config.flatDrop = 0.99999
54 mock_image_config.isTrimmed = True
56 self.dummy_exposure = FlatMock(config=mock_image_config).run()
57 self.detector = self.dummy_exposure.getDetector()
58 self.input_dims = {"detector": 0}
60 self.camera = FakeCamera([self.detector])
62 self.amp_names = []
63 for amp in self.detector:
64 self.amp_names.append(amp.getName())
66 def _create_ptc(self, amp_names, exp_times, means, ccobcurr=None, photo_charges=None):
67 """
68 Create a PTC with values for linearity tests.
70 Parameters
71 ----------
72 amp_names : `list` [`str`]
73 Names of amps.
74 exp_times : `np.ndarray`
75 Array of exposure times.
76 means : `np.ndarray`
77 Array of means.
78 ccobcurr : `np.ndarray`, optional
79 Array of CCOBCURR to put into auxiliary values.
80 photo_charges : `np.ndarray`, optional
81 Array of photoCharges to put into ptc.
83 Returns
84 -------
85 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`
86 PTC filled with relevant values.
87 """
88 exp_id_pairs = np.arange(len(exp_times)*2).reshape((len(exp_times), 2)).tolist()
90 if photo_charges is None:
91 photo_charges = np.full(len(exp_times), np.nan)
93 datasets = []
94 for i in range(len(exp_times)):
95 partial = PhotonTransferCurveDataset(amp_names, ptcFitType="PARTIAL", covMatrixSide=1)
96 for amp_name in amp_names:
97 # For the first amp, we add a few bad points.
98 if amp_name == amp_names[0] and i >= 5 and i < 7:
99 exp_id_mask = False
100 raw_mean = np.nan
101 else:
102 exp_id_mask = True
103 raw_mean = means[i]
105 partial.setAmpValuesPartialDataset(
106 amp_name,
107 inputExpIdPair=exp_id_pairs[i],
108 rawExpTime=exp_times[i],
109 rawMean=raw_mean,
110 rawVar=1.0,
111 kspValue=1.0,
112 expIdMask=exp_id_mask,
113 photoCharge=photo_charges[i],
114 )
116 if ccobcurr is not None:
117 partial.setAuxValuesPartialDataset({"CCOBCURR": ccobcurr[i]})
119 datasets.append(partial)
121 datasets.append(PhotonTransferCurveDataset(amp_names, ptcFitType="DUMMY"))
123 config = PhotonTransferCurveSolveTask.ConfigClass()
124 config.maximumRangeCovariancesAstier = 1
125 solve_task = PhotonTransferCurveSolveTask(config=config)
126 ptc = solve_task.run(datasets).outputPtcDataset
128 # Make the last amp a bad amp.
129 ptc.badAmps = [amp_names[-1]]
131 return ptc
133 def _check_linearity(self, linearity_type, min_adu=0.0, max_adu=100000.0):
134 """Run and check linearity.
136 Parameters
137 ----------
138 linearity_type : `str`
139 Must be ``Polynomial``, ``Squared``, or ``LookupTable``.
140 min_adu : `float`, optional
141 Minimum cut on ADU for fit.
142 max_adu : `float`, optional
143 Maximum cut on ADU for fit.
144 """
145 flux = 1000.
146 time_vec = np.arange(1., 101., 5)
147 k2_non_linearity = -5e-6
148 coeff = k2_non_linearity/(flux**2.)
150 mu_vec = flux * time_vec + k2_non_linearity * time_vec**2.
152 ptc = self._create_ptc(self.amp_names, time_vec, mu_vec)
154 config = LinearitySolveTask.ConfigClass()
155 config.linearityType = linearity_type
156 config.minLinearAdu = min_adu
157 config.maxLinearAdu = max_adu
159 task = LinearitySolveTask(config=config)
160 linearizer = task.run(ptc, [self.dummy_exposure], self.camera, self.input_dims).outputLinearizer
162 if linearity_type == "LookupTable":
163 t_max = config.maxLookupTableAdu / flux
164 time_range = np.linspace(0.0, t_max, config.maxLookupTableAdu)
165 signal_ideal = time_range * flux
166 signal_uncorrected = funcPolynomial(np.array([0.0, flux, k2_non_linearity]), time_range)
167 linearizer_table_row = signal_ideal - signal_uncorrected
169 # Skip the last amp which is marked bad.
170 for i, amp_name in enumerate(ptc.ampNames[:-1]):
171 if linearity_type in ["Squared", "Polynomial"]:
172 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][0], 0.0, atol=1e-2)
173 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][1], 1.0, rtol=1e-5)
174 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][2], coeff, rtol=1e-6)
176 if linearity_type == "Polynomial":
177 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][3], 0.0)
179 if linearity_type == "Squared":
180 self.assertEqual(len(linearizer.linearityCoeffs[amp_name]), 1)
181 self.assertFloatsAlmostEqual(linearizer.linearityCoeffs[amp_name][0], -coeff, rtol=1e-6)
182 else:
183 self.assertEqual(len(linearizer.linearityCoeffs[amp_name]), 2)
184 self.assertFloatsAlmostEqual(linearizer.linearityCoeffs[amp_name][0], -coeff, rtol=1e-6)
185 self.assertFloatsAlmostEqual(linearizer.linearityCoeffs[amp_name][1], 0.0)
187 else:
188 index = linearizer.linearityCoeffs[amp_name][0]
189 self.assertEqual(index, i)
190 self.assertEqual(len(linearizer.tableData[index, :]), len(linearizer_table_row))
191 self.assertFloatsAlmostEqual(linearizer.tableData[index, :], linearizer_table_row, rtol=1e-4)
193 lin_mask = np.isfinite(linearizer.fitResiduals[amp_name])
194 lin_mask_expected = (mu_vec > min_adu) & (mu_vec < max_adu) & ptc.expIdMask[amp_name]
196 self.assertListEqual(lin_mask.tolist(), lin_mask_expected.tolist())
197 self.assertFloatsAlmostEqual(linearizer.fitResiduals[amp_name][lin_mask], 0.0, atol=1e-2)
199 # If we apply the linearity correction, we should get the true
200 # linear values out.
201 image = lsst.afw.image.ImageF(len(mu_vec), 1)
202 image.array[:, :] = mu_vec
203 lin_func = linearizer.getLinearityTypeByName(linearizer.linearityType[amp_name])
204 lin_func()(
205 image,
206 coeffs=linearizer.linearityCoeffs[amp_name],
207 table=linearizer.tableData,
208 log=None,
209 )
211 linear_signal = flux * time_vec
212 self.assertFloatsAlmostEqual(image.array[0, :] / linear_signal, 1.0, rtol=1e-6)
214 def test_linearity_polynomial(self):
215 """Test linearity with polynomial fit."""
216 self._check_linearity("Polynomial")
218 def test_linearity_squared(self):
219 """Test linearity with a single order squared solution."""
220 self._check_linearity("Squared")
222 def test_linearity_table(self):
223 """Test linearity with a lookup table solution."""
224 self._check_linearity("LookupTable")
226 def test_linearity_polynomial_aducuts(self):
227 """Test linearity with polynomial and ADU cuts."""
228 self._check_linearity("Polynomial", min_adu=10000.0, max_adu=90000.0)
230 def _check_linearity_spline(self, do_pd_offsets=False, n_points=200):
231 """Check linearity with a spline solution.
233 Parameters
234 ----------
235 do_pd_offsets : `bool`, optional
236 Apply offsets to the photodiode data.
237 """
238 np.random.seed(12345)
240 # Create a test dataset representative of real data.
241 pd_values = np.linspace(1e-8, 2e-5, n_points)
242 time_values = pd_values * 1000000.
243 linear_ratio = 5e9
244 mu_linear = linear_ratio * pd_values
246 # Test spline parameters are taken from a test fit to LSSTCam
247 # data, run 7193D, detector 22, amp C00. The exact fit is not
248 # important, but this is only meant to be representative of
249 # the shape of the non-linearity that we see.
251 n_nodes = 10
253 non_lin_spline_nodes = np.linspace(0, mu_linear.max(), n_nodes)
254 non_lin_spline_values = np.array(
255 [0.0, -8.87, 1.46, 1.69, -6.92, -68.23, -78.01, -11.56, 80.26, 185.01]
256 )
258 spl = lsst.afw.math.makeInterpolate(
259 non_lin_spline_nodes,
260 non_lin_spline_values,
261 lsst.afw.math.stringToInterpStyle("AKIMA_SPLINE"),
262 )
264 mu_values = mu_linear + spl.interpolate(mu_linear)
265 mu_values += np.random.normal(scale=mu_values, size=len(mu_values)) / 10000.
267 # Add some outlier values.
268 if n_points >= 200:
269 outlier_indices = np.arange(5) + 170
270 else:
271 outlier_indices = []
272 mu_values[outlier_indices] += 200.0
274 # Add some small offsets to the pd_values if requested.
275 pd_values_offset = pd_values.copy()
276 ccobcurr = None
277 if do_pd_offsets:
278 ccobcurr = np.zeros(pd_values.size)
279 n_points_group = n_points//4
280 group0 = np.arange(n_points_group)
281 group1 = np.arange(n_points_group) + n_points_group
282 group2 = np.arange(n_points_group) + 2*n_points_group
283 group3 = np.arange(n_points_group) + 3*n_points_group
284 ccobcurr[group0] = 0.01
285 ccobcurr[group1] = 0.02
286 ccobcurr[group2] = 0.03
287 ccobcurr[group3] = 0.04
289 pd_offset_factors = [0.995, 1.0, 1.005, 1.002]
290 pd_values_offset[group0] *= pd_offset_factors[0]
291 pd_values_offset[group2] *= pd_offset_factors[2]
292 pd_values_offset[group3] *= pd_offset_factors[3]
294 # Add one bad photodiode value, but don't put it at the very
295 # end because that would change the spline node positions
296 # and make comparisons to the "truth" here in the tests
297 # more difficult.
298 pd_values_offset[-2] = np.nan
300 ptc = self._create_ptc(
301 self.amp_names,
302 time_values,
303 mu_values,
304 ccobcurr=ccobcurr,
305 photo_charges=pd_values_offset,
306 )
308 config = LinearitySolveTask.ConfigClass()
309 config.linearityType = "Spline"
310 config.usePhotodiode = True
311 config.minLinearAdu = 0.0
312 config.maxLinearAdu = np.nanmax(mu_values) + 1.0
313 config.splineKnots = n_nodes
314 config.splineGroupingMinPoints = 101
316 if do_pd_offsets:
317 config.splineGroupingColumn = "CCOBCURR"
319 task = LinearitySolveTask(config=config)
320 linearizer = task.run(
321 ptc,
322 [self.dummy_exposure],
323 self.camera,
324 self.input_dims,
325 ).outputLinearizer
327 # Skip the last amp which is marked bad.
328 for amp_name in ptc.ampNames[:-1]:
329 lin_mask = np.isfinite(linearizer.fitResiduals[amp_name])
331 # Make sure that anything in the input mask is still masked.
332 check, = np.where(~ptc.expIdMask[amp_name])
333 if len(check) > 0:
334 self.assertEqual(np.all(lin_mask[check]), False)
336 # Make sure the outliers are masked.
337 self.assertEqual(np.all(lin_mask[outlier_indices]), False)
339 # The first point at very low flux is noisier and so we exclude
340 # it from the test here.
341 self.assertFloatsAlmostEqual(
342 (linearizer.fitResiduals[amp_name][lin_mask] / mu_linear[lin_mask])[1:],
343 0.0,
344 atol=1.1e-3,
345 )
347 # If we apply the linearity correction, we should get the true
348 # linear values out.
349 image = lsst.afw.image.ImageF(len(mu_values), 1)
350 image.array[:, :] = mu_values
351 lin_func = linearizer.getLinearityTypeByName(linearizer.linearityType[amp_name])
352 lin_func()(
353 image,
354 coeffs=linearizer.linearityCoeffs[amp_name],
355 log=None,
356 )
358 # We scale by the median because of ambiguity in the overall
359 # gain parameter which is not part of the non-linearity.
360 ratio = image.array[0, lin_mask]/mu_linear[lin_mask]
361 self.assertFloatsAlmostEqual(
362 ratio / np.median(ratio),
363 1.0,
364 rtol=5e-4,
365 )
367 # Check that the spline parameters recovered are consistent,
368 # with input to some low-grade precision.
369 # The first element should be identically zero.
370 self.assertFloatsEqual(linearizer.linearityCoeffs[amp_name][0], 0.0)
372 # We have two different comparisons here; for the terms that are
373 # |value| < 20 (offset) or |value| > 20 (ratio), to avoid
374 # divide-by-small-number problems. In all cases these are
375 # approximate, and the real test is in the residuals.
376 small = (np.abs(non_lin_spline_values) < 20)
378 spline_atol = 6.0 if do_pd_offsets else 2.0
379 spline_rtol = 0.14 if do_pd_offsets else 0.05
381 self.assertFloatsAlmostEqual(
382 linearizer.linearityCoeffs[amp_name][n_nodes:][small],
383 non_lin_spline_values[small],
384 atol=spline_atol,
385 )
386 self.assertFloatsAlmostEqual(
387 linearizer.linearityCoeffs[amp_name][n_nodes:][~small],
388 non_lin_spline_values[~small],
389 rtol=spline_rtol,
390 )
392 # And check the offsets if they were included.
393 if do_pd_offsets:
394 # The relative scaling is to group 1.
395 fit_offset_factors = linearizer.fitParams[amp_name][1] / linearizer.fitParams[amp_name]
397 self.assertFloatsAlmostEqual(fit_offset_factors, np.array(pd_offset_factors), rtol=6e-4)
399 def test_linearity_spline(self):
400 self._check_linearity_spline()
402 def test_linearity_spline_offsets(self):
403 self._check_linearity_spline(do_pd_offsets=True)
405 def test_linearity_spline_offsets_too_few_points(self):
406 with self.assertRaisesRegex(RuntimeError, "too few points"):
407 self._check_linearity_spline(do_pd_offsets=True, n_points=100)
410class TestMemory(lsst.utils.tests.MemoryTestCase):
411 pass
414def setup_module(module):
415 lsst.utils.tests.init()
418if __name__ == "__main__": 418 ↛ 419line 418 didn't jump to line 419, because the condition on line 418 was never true
419 lsst.utils.tests.init()
420 unittest.main()