Coverage for tests/test_ptcDataset.py: 8%
234 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 04:06 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 04:06 -0700
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import unittest
22import tempfile
23import copy
24import logging
26import numpy as np
28import lsst.utils.tests
30from lsst.ip.isr import PhotonTransferCurveDataset
31import lsst.ip.isr.isrMock as isrMock
34class PtcDatasetCases(lsst.utils.tests.TestCase):
35 """Test that write/read methods of PhotonTransferCurveDataset work
36 """
37 def setUp(self):
39 self.flatMean = 2000
40 self.readNoiseAdu = 10
41 mockImageConfig = isrMock.IsrMock.ConfigClass()
43 # flatDrop is not really relevant as we replace the data
44 # but good to note it in case we change how this image is made
45 mockImageConfig.flatDrop = 0.99999
46 mockImageConfig.isTrimmed = True
48 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
49 self.flatExp2 = self.flatExp1.clone()
50 (shapeY, shapeX) = self.flatExp1.getDimensions()
52 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
54 self.rng1 = np.random.RandomState(1984)
55 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
56 self.rng2 = np.random.RandomState(666)
57 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
59 self.flatExp1.image.array[:] = flatData1
60 self.flatExp2.image.array[:] = flatData2
62 self.flux = 1000. # ADU/sec
63 self.gain = 1.5 # e-/ADU
64 self.noiseSq = 5*self.gain # 7.5 (e-)^2
65 self.c1 = 1./self.gain
66 self.timeVec = np.arange(1., 101., 5)
67 self.k2NonLinearity = -5e-6
68 # quadratic signal-chain non-linearity
69 muVec = self.flux*self.timeVec + self.k2NonLinearity*self.timeVec**2
71 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()]
72 self.dataset = PhotonTransferCurveDataset(self.ampNames, " ") # pack raw data for fitting
73 self.covariancesSqrtWeights = {}
74 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function
75 self.dataset.rawExpTimes[ampName] = self.timeVec
76 self.dataset.rawMeans[ampName] = muVec
77 self.covariancesSqrtWeights[ampName] = []
79 def _checkTypes(self, ptcDataset):
80 """Check that all the types are correct for a ptc dataset."""
81 for ampName in ptcDataset.ampNames:
82 self.assertIsInstance(ptcDataset.expIdMask[ampName], np.ndarray)
83 self.assertEqual(ptcDataset.expIdMask[ampName].dtype, bool)
84 self.assertIsInstance(ptcDataset.rawExpTimes[ampName], np.ndarray)
85 self.assertEqual(ptcDataset.rawExpTimes[ampName].dtype, np.float64)
86 self.assertIsInstance(ptcDataset.rawMeans[ampName], np.ndarray)
87 self.assertEqual(ptcDataset.rawMeans[ampName].dtype, np.float64)
88 self.assertIsInstance(ptcDataset.rawVars[ampName], np.ndarray)
89 self.assertEqual(ptcDataset.rawVars[ampName].dtype, np.float64)
90 self.assertEqual(ptcDataset.rowMeanVariance[ampName].dtype, np.float64)
91 self.assertIsInstance(ptcDataset.noiseList[ampName], np.ndarray)
92 self.assertEqual(ptcDataset.noiseList[ampName].dtype, np.float64)
93 self.assertIsInstance(ptcDataset.gain[ampName], float)
94 self.assertIsInstance(ptcDataset.gainErr[ampName], float)
95 self.assertIsInstance(ptcDataset.noise[ampName], float)
96 self.assertIsInstance(ptcDataset.noiseErr[ampName], float)
97 self.assertIsInstance(ptcDataset.histVars[ampName], np.ndarray)
98 self.assertEqual(ptcDataset.histVars[ampName].dtype, np.float64)
99 self.assertIsInstance(ptcDataset.histChi2Dofs[ampName], np.ndarray)
100 self.assertEqual(ptcDataset.histChi2Dofs[ampName].dtype, np.float64)
101 self.assertIsInstance(ptcDataset.kspValues[ampName], np.ndarray)
102 self.assertEqual(ptcDataset.kspValues[ampName].dtype, np.float64)
103 self.assertIsInstance(ptcDataset.ptcFitPars[ampName], np.ndarray)
104 self.assertEqual(ptcDataset.ptcFitPars[ampName].dtype, np.float64)
105 self.assertIsInstance(ptcDataset.ptcFitParsError[ampName], np.ndarray)
106 self.assertEqual(ptcDataset.ptcFitParsError[ampName].dtype, np.float64)
107 self.assertIsInstance(ptcDataset.ptcFitChiSq[ampName], float)
108 self.assertIsInstance(ptcDataset.ptcTurnoff[ampName], float)
109 self.assertIsInstance(ptcDataset.ptcTurnoffSamplingError[ampName], float)
110 self.assertIsInstance(ptcDataset.covariances[ampName], np.ndarray)
111 self.assertEqual(ptcDataset.covariances[ampName].dtype, np.float64)
112 self.assertIsInstance(ptcDataset.covariancesModel[ampName], np.ndarray)
113 self.assertEqual(ptcDataset.covariancesModel[ampName].dtype, np.float64)
114 self.assertIsInstance(ptcDataset.covariancesSqrtWeights[ampName], np.ndarray)
115 self.assertEqual(ptcDataset.covariancesSqrtWeights[ampName].dtype, np.float64)
116 self.assertIsInstance(ptcDataset.aMatrix[ampName], np.ndarray)
117 self.assertEqual(ptcDataset.aMatrix[ampName].dtype, np.float64)
118 self.assertIsInstance(ptcDataset.bMatrix[ampName], np.ndarray)
119 self.assertEqual(ptcDataset.bMatrix[ampName].dtype, np.float64)
120 self.assertIsInstance(ptcDataset.noiseMatrix[ampName], np.ndarray)
121 self.assertEqual(ptcDataset.noiseMatrix[ampName].dtype, np.float64)
122 self.assertIsInstance(ptcDataset.covariancesModelNoB[ampName], np.ndarray)
123 self.assertEqual(ptcDataset.covariancesModelNoB[ampName].dtype, np.float64)
124 self.assertIsInstance(ptcDataset.aMatrixNoB[ampName], np.ndarray)
125 self.assertEqual(ptcDataset.aMatrixNoB[ampName].dtype, np.float64)
126 self.assertIsInstance(ptcDataset.noiseMatrixNoB[ampName], np.ndarray)
127 self.assertEqual(ptcDataset.noiseMatrixNoB[ampName].dtype, np.float64)
128 self.assertIsInstance(ptcDataset.finalVars[ampName], np.ndarray)
129 self.assertEqual(ptcDataset.finalVars[ampName].dtype, np.float64)
130 self.assertIsInstance(ptcDataset.finalModelVars[ampName], np.ndarray)
131 self.assertEqual(ptcDataset.finalModelVars[ampName].dtype, np.float64)
132 self.assertIsInstance(ptcDataset.finalMeans[ampName], np.ndarray)
133 self.assertEqual(ptcDataset.finalMeans[ampName].dtype, np.float64)
134 self.assertIsInstance(ptcDataset.photoCharges[ampName], np.ndarray)
135 self.assertEqual(ptcDataset.photoCharges[ampName].dtype, np.float64)
137 for key, value in ptcDataset.auxValues.items():
138 self.assertIsInstance(value, np.ndarray)
139 self.assertEqual(value.dtype, np.float64)
141 def test_emptyPtcDataset(self):
142 """Test an empty PTC dataset."""
143 emptyDataset = PhotonTransferCurveDataset(
144 self.ampNames,
145 ptcFitType="PARTIAL",
146 )
147 self._checkTypes(emptyDataset)
149 with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
150 usedFilename = emptyDataset.writeText(f.name)
151 fromText = PhotonTransferCurveDataset.readText(usedFilename)
152 self.assertEqual(emptyDataset, fromText)
153 self._checkTypes(emptyDataset)
155 with tempfile.NamedTemporaryFile(suffix=".fits") as f:
156 usedFilename = emptyDataset.writeFits(f.name)
157 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
158 self.assertEqual(emptyDataset, fromFits)
159 self._checkTypes(emptyDataset)
161 def test_partialPtcDataset(self):
162 """Test of a partial PTC dataset."""
163 # Fill the dataset with made up data.
164 nSideCovMatrix = 2
165 nSideCovMatrixFullCovFit = 2
167 partialDataset = PhotonTransferCurveDataset(
168 self.ampNames,
169 ptcFitType="PARTIAL",
170 covMatrixSide=nSideCovMatrix,
171 covMatrixSideFullCovFit=nSideCovMatrixFullCovFit
172 )
173 self._checkTypes(partialDataset)
175 for ampName in partialDataset.ampNames:
176 partialDataset.setAmpValuesPartialDataset(
177 ampName,
178 inputExpIdPair=(10, 11),
179 rawExpTime=10.0,
180 rawMean=10.0,
181 rawVar=10.0,
182 )
184 for useAuxValues in [False, True]:
185 if useAuxValues:
186 partialDataset.setAuxValuesPartialDataset(
187 {
188 "CCOBCURR": 1.0,
189 "CCDTEMP": 0.0,
190 }
191 )
192 self._checkTypes(partialDataset)
194 with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
195 usedFilename = partialDataset.writeText(f.name)
196 fromText = PhotonTransferCurveDataset.readText(usedFilename)
197 self.assertEqual(fromText, partialDataset)
198 self._checkTypes(fromText)
200 with tempfile.NamedTemporaryFile(suffix=".fits") as f:
201 usedFilename = partialDataset.writeFits(f.name)
202 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
203 self.assertEqual(fromFits, partialDataset)
204 self._checkTypes(fromFits)
206 def test_ptcDatset(self):
207 """Test of a full PTC dataset."""
208 # Fill the dataset with made up data.
209 nSignalPoints = 5
210 nSideCovMatrixInput = 3 # Size of measured covariances
212 for nSideCovMatrixFullCovFitInput in np.arange(1, nSideCovMatrixInput + 2):
213 for fitType in ['POLYNOMIAL', 'EXPAPPROXIMATION', 'FULLCOVARIANCE']:
214 localDataset = PhotonTransferCurveDataset(
215 self.ampNames,
216 ptcFitType=fitType,
217 covMatrixSide=nSideCovMatrixInput,
218 covMatrixSideFullCovFit=nSideCovMatrixFullCovFitInput,
219 )
220 nSideCovMatrix = localDataset.covMatrixSide
221 nSideCovMatrixFullCovFit = localDataset.covMatrixSideFullCovFit
222 localDataset.badAmps = [localDataset.ampNames[0], localDataset.ampNames[1]]
223 for ampName in localDataset.ampNames:
225 localDataset.inputExpIdPairs[ampName] = [(1, 2)]*nSignalPoints
226 localDataset.expIdMask[ampName] = np.ones(nSignalPoints, dtype=bool)
227 localDataset.expIdMask[ampName][1] = False
228 localDataset.rawExpTimes[ampName] = np.arange(nSignalPoints, dtype=np.float64)
229 localDataset.rawMeans[ampName] = self.flux*np.arange(nSignalPoints)
230 localDataset.rawVars[ampName] = self.c1*self.flux*np.arange(nSignalPoints)
231 localDataset.photoCharges[ampName] = np.full(nSignalPoints, np.nan)
232 localDataset.gain[ampName] = self.gain
233 localDataset.gainErr[ampName] = 0.1
234 localDataset.noise[ampName] = self.noiseSq
235 localDataset.noiseErr[ampName] = 2.0
236 localDataset.histVars[ampName] = localDataset.rawVars[ampName]
237 localDataset.histChi2Dofs[ampName] = np.full(nSignalPoints, 1.0)
238 localDataset.kspValues[ampName] = np.full(nSignalPoints, 0.5)
240 localDataset.finalVars[ampName] = self.c1*self.flux*np.arange(nSignalPoints)
241 localDataset.finalModelVars[ampName] = np.full(nSignalPoints, 100.0)
242 localDataset.finalMeans[ampName] = self.flux*np.arange(nSignalPoints)
244 if fitType in ['POLYNOMIAL', 'EXPAPPROXIMATION', ]:
245 localDataset.ptcFitPars[ampName] = np.array([10.0, 1.5, 1e-6])
246 localDataset.ptcFitParsError[ampName] = np.array([1.0, 0.2, 1e-7])
247 localDataset.ptcFitChiSq[ampName] = 1.0
248 localDataset.ptcTurnoff[ampName] = localDataset.rawMeans[ampName][-1]
249 localDataset.ptcTurnoffSamplingError[ampName] = localDataset.ptcTurnoff[ampName]/100.
251 localDataset.covariances[ampName] = np.full(
252 (nSignalPoints, nSideCovMatrix, nSideCovMatrix), 105.0)
253 localDataset.covariancesModel[ampName] = np.full(
254 (nSignalPoints, nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan)
255 localDataset.covariancesSqrtWeights[ampName] = np.full((nSignalPoints, nSideCovMatrix,
256 nSideCovMatrix), 10.0)
257 localDataset.aMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
258 nSideCovMatrixFullCovFit), np.nan)
259 localDataset.bMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
260 nSideCovMatrixFullCovFit), np.nan)
261 localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
262 nSideCovMatrixFullCovFit), np.nan)
263 localDataset.covariancesModelNoB[ampName] = np.full((nSignalPoints,
264 nSideCovMatrixFullCovFit,
265 nSideCovMatrixFullCovFit), np.nan)
266 localDataset.aMatrixNoB[ampName] = np.full(
267 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan)
268 localDataset.noiseMatrixNoB[ampName] = np.full(
269 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan)
271 if localDataset.ptcFitType in ['FULLCOVARIANCE', ]:
272 localDataset.ptcFitPars[ampName] = np.array([np.nan, np.nan])
273 localDataset.ptcFitParsError[ampName] = np.array([np.nan, np.nan])
274 localDataset.ptcFitChiSq[ampName] = np.nan
275 localDataset.ptcTurnoff[ampName] = np.nan
276 localDataset.ptcTurnoffSamplingError[ampName] = np.nan
278 localDataset.covariances[ampName] = np.full(
279 (nSignalPoints, nSideCovMatrix, nSideCovMatrix), 105.0)
280 localDataset.covariancesModel[ampName] = np.full(
281 (nSignalPoints, nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 100.0)
282 localDataset.covariancesSqrtWeights[ampName] = np.full((nSignalPoints, nSideCovMatrix,
283 nSideCovMatrix), 10.0)
284 localDataset.aMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
285 nSideCovMatrixFullCovFit), 1e-6)
286 localDataset.bMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
287 nSideCovMatrixFullCovFit), 1e-7)
288 localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
289 nSideCovMatrixFullCovFit), 3.0)
290 localDataset.covariancesModelNoB[ampName] = np.full((nSignalPoints,
291 nSideCovMatrixFullCovFit,
292 nSideCovMatrixFullCovFit), 15.0)
293 localDataset.aMatrixNoB[ampName] = np.full(
294 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 2e-6)
295 localDataset.noiseMatrixNoB[ampName] = np.full(
296 (nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 3.0)
298 for useAuxValues in [False, True]:
299 if useAuxValues:
300 localDataset.auxValues = {
301 "CCOBCURR": np.ones(nSignalPoints),
302 "CCDTEMP": np.zeros(nSignalPoints),
303 }
305 self._checkTypes(localDataset)
306 with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
307 usedFilename = localDataset.writeText(f.name)
308 fromText = PhotonTransferCurveDataset.readText(usedFilename)
309 self.assertEqual(fromText, localDataset)
310 self._checkTypes(fromText)
312 with tempfile.NamedTemporaryFile(suffix=".fits") as f:
313 usedFilename = localDataset.writeFits(f.name)
314 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
315 self.assertEqual(fromFits, localDataset)
316 self._checkTypes(fromFits)
318 def test_getExpIdsUsed(self):
319 localDataset = copy.copy(self.dataset)
321 for pair in [(12, 34), (56, 78), (90, 10)]:
322 localDataset.inputExpIdPairs["C:0,0"].append(pair)
323 localDataset.expIdMask["C:0,0"] = np.array([True, False, True])
324 self.assertTrue(np.all(localDataset.getExpIdsUsed("C:0,0") == [(12, 34), (90, 10)]))
326 localDataset.expIdMask["C:0,0"] = np.array([True, False, True, True]) # wrong length now
327 with self.assertRaises(AssertionError):
328 localDataset.getExpIdsUsed("C:0,0")
330 def test_getGoodAmps(self):
331 dataset = self.dataset
333 self.assertTrue(dataset.ampNames == self.ampNames)
334 dataset.badAmps.append("C:0,1")
335 self.assertTrue(dataset.getGoodAmps() == [amp for amp in self.ampNames if amp != "C:0,1"])
337 def test_ptcDataset_pre_dm38309(self):
338 """Test for PTC datasets created by cpSolvePtcTask prior to DM-38309.
339 """
340 localDataset = copy.copy(self.dataset)
342 for pair in [[(12, 34)], [(56, 78)], [(90, 10)]]:
343 localDataset.inputExpIdPairs["C:0,0"].append(pair)
344 localDataset.expIdMask["C:0,0"] = np.array([True, False, True])
346 with self.assertLogs("lsst.ip.isr.calibType", logging.WARNING) as cm:
347 used = localDataset.getExpIdsUsed("C:0,0")
348 self.assertIn("PTC file was written incorrectly", cm.output[0])
350 self.assertTrue(np.all(used == [(12, 34), (90, 10)]))
353class MemoryTester(lsst.utils.tests.MemoryTestCase):
354 pass
357def setup_module(module):
358 lsst.utils.tests.init()
361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true
362 import sys
363 setup_module(sys.modules[__name__])
364 unittest.main()