Coverage for tests/test_linearity.py: 14%
188 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-06 03:59 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-06 03:59 -0700
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe linearity code."""
27import unittest
28import numpy as np
30import lsst.utils
31import lsst.utils.tests
33from lsst.ip.isr import PhotonTransferCurveDataset, PhotodiodeCalib
35import lsst.afw.image
36import lsst.afw.math
37from lsst.cp.pipe import LinearitySolveTask
38from lsst.cp.pipe.ptc import PhotonTransferCurveSolveTask
39from lsst.cp.pipe.utils import funcPolynomial
40from lsst.ip.isr.isrMock import FlatMock, IsrMock
43class FakeCamera(list):
44 def getName(self):
45 return "FakeCam"
48class LinearityTaskTestCase(lsst.utils.tests.TestCase):
49 """Test case for the linearity tasks."""
51 def setUp(self):
52 mock_image_config = IsrMock.ConfigClass()
53 mock_image_config.flatDrop = 0.99999
54 mock_image_config.isTrimmed = True
56 self.dummy_exposure = FlatMock(config=mock_image_config).run()
57 self.detector = self.dummy_exposure.getDetector()
58 self.input_dims = {"detector": 0}
60 self.camera = FakeCamera([self.detector])
62 self.amp_names = []
63 for amp in self.detector:
64 self.amp_names.append(amp.getName())
66 def _create_ptc(self, amp_names, exp_times, means, ccobcurr=None):
67 """
68 Create a PTC with values for linearity tests.
70 Parameters
71 ----------
72 amp_names : `list` [`str`]
73 Names of amps.
74 exp_times : `np.ndarray`
75 Array of exposure times.
76 means : `np.ndarray`
77 Array of means.
78 ccobcurr : `np.ndarray`, optional
79 Array of CCOBCURR to put into auxiliary values.
81 Returns
82 -------
83 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`
84 PTC filled with relevant values.
85 """
86 exp_id_pairs = np.arange(len(exp_times)*2).reshape((len(exp_times), 2)).tolist()
88 datasets = []
89 for i in range(len(exp_times)):
90 partial = PhotonTransferCurveDataset(amp_names, ptcFitType="PARTIAL", covMatrixSide=1)
91 for amp_name in amp_names:
92 # For the first amp, we add a few bad points.
93 if amp_name == amp_names[0] and i >= 5 and i < 7:
94 exp_id_mask = False
95 else:
96 exp_id_mask = True
98 partial.setAmpValuesPartialDataset(
99 amp_name,
100 inputExpIdPair=exp_id_pairs[i],
101 rawExpTime=exp_times[i],
102 rawMean=means[i],
103 rawVar=1.0,
104 kspValue=1.0,
105 expIdMask=exp_id_mask,
106 )
108 if ccobcurr is not None:
109 partial.setAuxValuesPartialDataset({"CCOBCURR": ccobcurr[i]})
111 datasets.append(partial)
113 datasets.append(PhotonTransferCurveDataset(amp_names, ptcFitType="DUMMY"))
115 config = PhotonTransferCurveSolveTask.ConfigClass()
116 config.maximumRangeCovariancesAstier = 1
117 solve_task = PhotonTransferCurveSolveTask(config=config)
118 ptc = solve_task.run(datasets).outputPtcDataset
120 # Make the last amp a bad amp.
121 ptc.badAmps = [amp_names[-1]]
123 return ptc
125 def _check_linearity(self, linearity_type, min_adu=0.0, max_adu=100000.0):
126 """Run and check linearity.
128 Parameters
129 ----------
130 linearity_type : `str`
131 Must be ``Polynomial``, ``Squared``, or ``LookupTable``.
132 min_adu : `float`, optional
133 Minimum cut on ADU for fit.
134 max_adu : `float`, optional
135 Maximum cut on ADU for fit.
136 """
137 flux = 1000.
138 time_vec = np.arange(1., 101., 5)
139 k2_non_linearity = -5e-6
140 coeff = k2_non_linearity/(flux**2.)
142 mu_vec = flux * time_vec + k2_non_linearity * time_vec**2.
144 ptc = self._create_ptc(self.amp_names, time_vec, mu_vec)
146 config = LinearitySolveTask.ConfigClass()
147 config.linearityType = linearity_type
148 config.minLinearAdu = min_adu
149 config.maxLinearAdu = max_adu
151 task = LinearitySolveTask(config=config)
152 linearizer = task.run(ptc, [self.dummy_exposure], self.camera, self.input_dims).outputLinearizer
154 if linearity_type == "LookupTable":
155 t_max = config.maxLookupTableAdu / flux
156 time_range = np.linspace(0.0, t_max, config.maxLookupTableAdu)
157 signal_ideal = time_range * flux
158 signal_uncorrected = funcPolynomial(np.array([0.0, flux, k2_non_linearity]), time_range)
159 linearizer_table_row = signal_ideal - signal_uncorrected
161 # Skip the last amp which is marked bad.
162 for i, amp_name in enumerate(ptc.ampNames[:-1]):
163 if linearity_type in ["Squared", "Polynomial"]:
164 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][0], 0.0, atol=1e-2)
165 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][1], 1.0, rtol=1e-5)
166 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][2], coeff, rtol=1e-6)
168 if linearity_type == "Polynomial":
169 self.assertFloatsAlmostEqual(linearizer.fitParams[amp_name][3], 0.0)
171 if linearity_type == "Squared":
172 self.assertEqual(len(linearizer.linearityCoeffs[amp_name]), 1)
173 self.assertFloatsAlmostEqual(linearizer.linearityCoeffs[amp_name][0], -coeff, rtol=1e-6)
174 else:
175 self.assertEqual(len(linearizer.linearityCoeffs[amp_name]), 2)
176 self.assertFloatsAlmostEqual(linearizer.linearityCoeffs[amp_name][0], -coeff, rtol=1e-6)
177 self.assertFloatsAlmostEqual(linearizer.linearityCoeffs[amp_name][1], 0.0)
179 else:
180 index = linearizer.linearityCoeffs[amp_name][0]
181 self.assertEqual(index, i)
182 self.assertEqual(len(linearizer.tableData[index, :]), len(linearizer_table_row))
183 self.assertFloatsAlmostEqual(linearizer.tableData[index, :], linearizer_table_row, rtol=1e-4)
185 lin_mask = np.isfinite(linearizer.fitResiduals[amp_name])
186 lin_mask_expected = (mu_vec > min_adu) & (mu_vec < max_adu) & ptc.expIdMask[amp_name]
188 self.assertListEqual(lin_mask.tolist(), lin_mask_expected.tolist())
189 self.assertFloatsAlmostEqual(linearizer.fitResiduals[amp_name][lin_mask], 0.0, atol=1e-2)
191 # If we apply the linearity correction, we should get the true
192 # linear values out.
193 image = lsst.afw.image.ImageF(len(mu_vec), 1)
194 image.array[:, :] = mu_vec
195 lin_func = linearizer.getLinearityTypeByName(linearizer.linearityType[amp_name])
196 lin_func()(
197 image,
198 coeffs=linearizer.linearityCoeffs[amp_name],
199 table=linearizer.tableData,
200 log=None,
201 )
203 linear_signal = flux * time_vec
204 self.assertFloatsAlmostEqual(image.array[0, :] / linear_signal, 1.0, rtol=1e-6)
206 def test_linearity_polynomial(self):
207 """Test linearity with polynomial fit."""
208 self._check_linearity("Polynomial")
210 def test_linearity_squared(self):
211 """Test linearity with a single order squared solution."""
212 self._check_linearity("Squared")
214 def test_linearity_table(self):
215 """Test linearity with a lookup table solution."""
216 self._check_linearity("LookupTable")
218 def test_linearity_polynomial_aducuts(self):
219 """Test linearity with polynomial and ADU cuts."""
220 self._check_linearity("Polynomial", min_adu=10000.0, max_adu=90000.0)
222 def _check_linearity_spline(self, do_pd_offsets=False):
223 """Check linearity with a spline solution.
225 Parameters
226 ----------
227 do_pd_offsets : `bool`, optional
228 Apply offsets to the photodiode data.
229 """
230 np.random.seed(12345)
232 # Create a test dataset representative of real data.
233 pd_values = np.linspace(1e-8, 2e-5, 200)
234 time_values = pd_values * 1000000.
235 linear_ratio = 5e9
236 mu_linear = linear_ratio * pd_values
238 # Test spline parameters are taken from a test fit to LSSTCam
239 # data, run 7193D, detector 22, amp C00. The exact fit is not
240 # important, but this is only meant to be representative of
241 # the shape of the non-linearity that we see.
243 n_nodes = 10
245 non_lin_spline_nodes = np.linspace(0, mu_linear.max(), n_nodes)
246 non_lin_spline_values = np.array(
247 [0.0, -8.87, 1.46, 1.69, -6.92, -68.23, -78.01, -11.56, 80.26, 185.01]
248 )
250 spl = lsst.afw.math.makeInterpolate(
251 non_lin_spline_nodes,
252 non_lin_spline_values,
253 lsst.afw.math.stringToInterpStyle("AKIMA_SPLINE"),
254 )
256 mu_values = mu_linear + spl.interpolate(mu_linear)
257 mu_values += np.random.normal(scale=mu_values, size=len(mu_values)) / 10000.
259 # Add some outlier values.
260 outlier_indices = np.arange(5) + 170
261 mu_values[outlier_indices] += 200.0
263 # Add some small offsets to the pd_values if requested.
264 pd_values_offset = pd_values.copy()
265 ccobcurr = None
266 if do_pd_offsets:
267 ccobcurr = np.zeros(pd_values.size)
268 group0 = np.arange(50)
269 group1 = np.arange(50) + 50
270 group2 = np.arange(50) + 100
271 group3 = np.arange(50) + 150
272 ccobcurr[group0] = 0.01
273 ccobcurr[group1] = 0.02
274 ccobcurr[group2] = 0.03
275 ccobcurr[group3] = 0.04
277 pd_offset_factors = [0.995, 1.0, 1.005, 1.002]
278 pd_values_offset[group0] *= pd_offset_factors[0]
279 pd_values_offset[group2] *= pd_offset_factors[2]
280 pd_values_offset[group3] *= pd_offset_factors[3]
282 ptc = self._create_ptc(self.amp_names, time_values, mu_values, ccobcurr=ccobcurr)
284 # And create a bunch of PD datasets.
285 amp_name = ptc.ampNames[0]
286 exp_id_pairs = ptc.inputExpIdPairs[amp_name]
288 pd_handles = []
290 for i, exp_id_pair in enumerate(exp_id_pairs):
291 time_samples = np.linspace(0, 20.0, 100)
292 current_samples = np.zeros(100)
293 current_samples[50] = -1.0*pd_values_offset[i]
295 pd_calib = PhotodiodeCalib(timeSamples=time_samples, currentSamples=current_samples)
296 pd_calib.currentScale = -1.0
297 pd_calib.integrationMethod = "CHARGE_SUM"
299 pd_handles.append(
300 lsst.pipe.base.InMemoryDatasetHandle(
301 pd_calib,
302 dataId={"exposure": exp_id_pair[0]},
303 )
304 )
305 pd_handles.append(
306 lsst.pipe.base.InMemoryDatasetHandle(
307 pd_calib,
308 dataId={"exposure": exp_id_pair[1]},
309 )
310 )
312 config = LinearitySolveTask.ConfigClass()
313 config.linearityType = "Spline"
314 config.usePhotodiode = True
315 config.photodiodeIntegrationMethod = "CHARGE_SUM"
316 config.minLinearAdu = 0.0
317 config.maxLinearAdu = np.max(mu_values) + 1.0
318 config.splineKnots = n_nodes
320 if do_pd_offsets:
321 config.splineGroupingColumn = "CCOBCURR"
323 task = LinearitySolveTask(config=config)
324 linearizer = task.run(
325 ptc,
326 [self.dummy_exposure],
327 self.camera,
328 self.input_dims,
329 inputPhotodiodeData=pd_handles,
330 ).outputLinearizer
332 # Skip the last amp which is marked bad.
333 for amp_name in ptc.ampNames[:-1]:
334 lin_mask = np.isfinite(linearizer.fitResiduals[amp_name])
336 # Make sure that anything in the input mask is still masked.
337 check, = np.where(~ptc.expIdMask[amp_name])
338 if len(check) > 0:
339 self.assertEqual(np.all(lin_mask[check]), False)
341 # Make sure the outliers are masked.
342 self.assertEqual(np.all(lin_mask[outlier_indices]), False)
344 # The first point at very low flux is noisier and so we exclude
345 # it from the test here.
346 self.assertFloatsAlmostEqual(
347 (linearizer.fitResiduals[amp_name][lin_mask] / mu_linear[lin_mask])[1:],
348 0.0,
349 atol=1e-3,
350 )
352 # If we apply the linearity correction, we should get the true
353 # linear values out.
354 image = lsst.afw.image.ImageF(len(mu_values), 1)
355 image.array[:, :] = mu_values
356 lin_func = linearizer.getLinearityTypeByName(linearizer.linearityType[amp_name])
357 lin_func()(
358 image,
359 coeffs=linearizer.linearityCoeffs[amp_name],
360 log=None,
361 )
363 # We scale by the median because of ambiguity in the overall
364 # gain parameter which is not part of the non-linearity.
365 ratio = image.array[0, lin_mask]/mu_linear[lin_mask]
366 self.assertFloatsAlmostEqual(
367 ratio / np.median(ratio),
368 1.0,
369 rtol=5e-4,
370 )
372 # Check that the spline parameters recovered are consistent,
373 # with input to some low-grade precision.
374 # The first element should be identically zero.
375 self.assertFloatsEqual(linearizer.linearityCoeffs[amp_name][0], 0.0)
377 # We have two different comparisons here; for the terms that are
378 # |value| < 20 (offset) or |value| > 20 (ratio), to avoid
379 # divide-by-small-number problems. In all cases these are
380 # approximate, and the real test is in the residuals.
381 small = (np.abs(non_lin_spline_values) < 20)
383 spline_atol = 5.0 if do_pd_offsets else 2.0
384 spline_rtol = 0.1 if do_pd_offsets else 0.05
386 self.assertFloatsAlmostEqual(
387 linearizer.linearityCoeffs[amp_name][n_nodes:][small],
388 non_lin_spline_values[small],
389 atol=spline_atol,
390 )
391 self.assertFloatsAlmostEqual(
392 linearizer.linearityCoeffs[amp_name][n_nodes:][~small],
393 non_lin_spline_values[~small],
394 rtol=spline_rtol,
395 )
397 # And check the offsets if they were included.
398 if do_pd_offsets:
399 # The relative scaling is to group 1.
400 fit_offset_factors = linearizer.fitParams[amp_name][1] / linearizer.fitParams[amp_name]
402 self.assertFloatsAlmostEqual(fit_offset_factors, np.array(pd_offset_factors), rtol=6e-4)
404 def test_linearity_spline(self):
405 self._check_linearity_spline()
407 def test_linearity_spline_offsets(self):
408 self._check_linearity_spline(do_pd_offsets=True)
411class TestMemory(lsst.utils.tests.MemoryTestCase):
412 pass
415def setup_module(module):
416 lsst.utils.tests.init()
419if __name__ == "__main__": 419 ↛ 420line 419 didn't jump to line 420, because the condition on line 419 was never true
420 lsst.utils.tests.init()
421 unittest.main()