Coverage for tests/test_ptc.py: 8%
385 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-20 02:19 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-20 02:19 -0700
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe."""
27import unittest
28import numpy as np
29import copy
30import tempfile
31import logging
33import lsst.utils
34import lsst.utils.tests
36import lsst.cp.pipe as cpPipe
37import lsst.ip.isr.isrMock as isrMock
38from lsst.ip.isr import PhotonTransferCurveDataset
39from lsst.cp.pipe.utils import (funcPolynomial, makeMockFlats)
41from lsst.pipe.base import TaskMetadata
44class FakeCamera(list):
45 def getName(self):
46 return "FakeCam"
49class PretendRef():
50 "A class to act as a mock exposure reference"
51 def __init__(self, exposure):
52 self.exp = exposure
54 def get(self, component=None):
55 if component == 'visitInfo':
56 return self.exp.getVisitInfo()
57 elif component == 'detector':
58 return self.exp.getDetector()
59 elif component == 'metadata':
60 return self.exp.getMetadata()
61 else:
62 return self.exp
65class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase):
66 """A test case for the PTC tasks."""
68 def setUp(self):
69 self.defaultConfigExtract = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
70 self.defaultTaskExtract = cpPipe.ptc.PhotonTransferCurveExtractTask(config=self.defaultConfigExtract)
72 self.defaultConfigSolve = cpPipe.ptc.PhotonTransferCurveSolveTask.ConfigClass()
73 self.defaultTaskSolve = cpPipe.ptc.PhotonTransferCurveSolveTask(config=self.defaultConfigSolve)
75 self.flatMean = 2000
76 self.readNoiseAdu = 10
77 mockImageConfig = isrMock.IsrMock.ConfigClass()
79 # flatDrop is not really relevant as we replace the data
80 # but good to note it in case we change how this image is made
81 mockImageConfig.flatDrop = 0.99999
82 mockImageConfig.isTrimmed = True
84 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
85 self.flatExp2 = self.flatExp1.clone()
86 (shapeY, shapeX) = self.flatExp1.getDimensions()
88 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
90 self.rng1 = np.random.RandomState(1984)
91 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
92 self.rng2 = np.random.RandomState(666)
93 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
95 self.flatExp1.image.array[:] = flatData1
96 self.flatExp2.image.array[:] = flatData2
98 # create fake PTC data to see if fit works, for one amp ('amp')
99 self.flux = 1000. # ADU/sec
100 self.timeVec = np.arange(1., 101., 5)
101 self.k2NonLinearity = -5e-6
102 # quadratic signal-chain non-linearity
103 muVec = self.flux*self.timeVec + self.k2NonLinearity*self.timeVec**2
104 self.gain = 0.75 # e-/ADU
105 self.c1 = 1./self.gain
106 self.noiseSq = 2*self.gain # 7.5 (e-)^2
107 self.a00 = -1.2e-6
108 self.c2 = -1.5e-6
109 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean
111 self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()]
112 self.dataset = PhotonTransferCurveDataset(self.ampNames, ptcFitType="PARTIAL")
113 self.covariancesSqrtWeights = {}
114 for ampName in self.ampNames: # just the expTimes and means here - vars vary per function
115 self.dataset.rawExpTimes[ampName] = self.timeVec
116 self.dataset.rawMeans[ampName] = muVec
117 self.dataset.covariancesSqrtWeights[ampName] = np.zeros((1,
118 self.dataset.covMatrixSide,
119 self.dataset.covMatrixSide))
121 # ISR metadata
122 self.metadataContents = TaskMetadata()
123 self.metadataContents["isr"] = {}
124 # Overscan readout noise [in ADU]
125 for amp in self.ampNames:
126 self.metadataContents["isr"][f"RESIDUAL STDEV {amp}"] = np.sqrt(self.noiseSq)/self.gain
128 def test_covAstier(self):
129 """Test to check getCovariancesAstier
131 We check that the gain is the same as the imput gain from the
132 mock data, that the covariances via FFT (as it is in
133 MeasurePhotonTransferCurveTask when doCovariancesAstier=True)
134 are the same as calculated in real space, and that Cov[0, 0]
135 (i.e., the variances) are similar to the variances calculated
136 with the standard method (when doCovariancesAstier=false),
138 """
139 extractConfig = self.defaultConfigExtract
140 extractConfig.minNumberGoodPixelsForCovariance = 5000
141 extractConfig.detectorMeasurementRegion = 'FULL'
142 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig)
144 solveConfig = self.defaultConfigSolve
145 solveConfig.ptcFitType = 'FULLCOVARIANCE'
146 # Cut off the low-flux point which is a bad fit, and this
147 # also exercises this functionality and makes the tests
148 # run a lot faster.
149 solveConfig.minMeanSignal["ALL_AMPS"] = 2000.0
150 # Set the outlier fit threshold higher than the default appropriate
151 # for this test dataset.
152 solveConfig.maxSignalInitialPtcOutlierFit = 90000.0
153 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=solveConfig)
155 inputGain = self.gain
157 muStandard, varStandard = {}, {}
158 expDict = {}
159 expIds = []
160 idCounter = 0
161 for expTime in self.timeVec:
162 mockExp1, mockExp2 = makeMockFlats(expTime, gain=inputGain,
163 readNoiseElectrons=3,
164 expId1=idCounter, expId2=idCounter+1)
165 mockExpRef1 = PretendRef(mockExp1)
166 mockExpRef2 = PretendRef(mockExp2)
167 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter+1))
168 expIds.append(idCounter)
169 expIds.append(idCounter+1)
170 for ampNumber, ampName in enumerate(self.ampNames):
171 # cov has (i, j, var, cov, npix)
172 im1Area, im2Area, imStatsCtrl, mu1, mu2 = extractTask.getImageAreasMasksStats(mockExp1,
173 mockExp2)
174 muDiff, varDiff, covAstier = extractTask.measureMeanVarCov(im1Area, im2Area, imStatsCtrl,
175 mu1, mu2)
176 muStandard.setdefault(ampName, []).append(muDiff)
177 varStandard.setdefault(ampName, []).append(varDiff)
178 idCounter += 2
180 resultsExtract = extractTask.run(inputExp=expDict, inputDims=expIds,
181 taskMetadata=[self.metadataContents for x in expIds])
183 # Force the last PTC dataset to have a NaN, and ensure that the
184 # task runs (DM-38029). This is a minor perturbation and does not
185 # affect the output comparison. Note that we use index -2 because
186 # these datasets are in pairs of [real, dummy] to match the inputs
187 # to the extract task.
188 resultsExtract.outputCovariances[-2].rawMeans['C:0,0'] = np.array([np.nan])
189 resultsExtract.outputCovariances[-2].rawVars['C:0,0'] = np.array([np.nan])
191 # Force the next-to-last PTC dataset to have a decreased variance to
192 # ensure that the outlier fit rejection works. Note that we use
193 # index -4 because these datasets are in pairs of [real, dummy] to
194 # match the inputs to the extract task.
195 rawVar = resultsExtract.outputCovariances[-4].rawVars['C:0,0']
196 resultsExtract.outputCovariances[-4].rawVars['C:0,0'] = rawVar*0.9
198 # Reorganize the outputCovariances so we can confirm they come
199 # out sorted afterwards.
200 outputCovariancesRev = resultsExtract.outputCovariances[::-1]
202 resultsSolve = solveTask.run(outputCovariancesRev,
203 camera=FakeCamera([self.flatExp1.getDetector()]))
205 ptc = resultsSolve.outputPtcDataset
207 for amp in self.ampNames:
208 self.assertAlmostEqual(ptc.gain[amp], inputGain, places=2)
209 for v1, v2 in zip(varStandard[amp], ptc.finalVars[amp]):
210 self.assertAlmostEqual(v1/v2, 1.0, places=1)
212 # Check that the PTC turnoff is correctly computed.
213 # This will be different for the C:0,0 amp.
214 if amp == 'C:0,0':
215 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-3])
216 else:
217 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-1])
219 # Test that all the quantities are correctly ordered and have
220 # not accidentally been masked. We check every other output ([::2])
221 # because these datasets are in pairs of [real, dummy] to
222 # match the inputs to the extract task.
223 for i, extractPtc in enumerate(resultsExtract.outputCovariances[::2]):
224 self.assertFloatsAlmostEqual(
225 extractPtc.rawExpTimes[ampName][0],
226 ptc.rawExpTimes[ampName][i],
227 )
228 self.assertFloatsAlmostEqual(
229 extractPtc.rawMeans[ampName][0],
230 ptc.rawMeans[ampName][i],
231 )
232 self.assertFloatsAlmostEqual(
233 extractPtc.rawVars[ampName][0],
234 ptc.rawVars[ampName][i],
235 )
236 self.assertFloatsAlmostEqual(
237 extractPtc.histVars[ampName][0],
238 ptc.histVars[ampName][i],
239 )
240 self.assertFloatsAlmostEqual(
241 extractPtc.histChi2Dofs[ampName][0],
242 ptc.histChi2Dofs[ampName][i],
243 )
244 self.assertFloatsAlmostEqual(
245 extractPtc.kspValues[ampName][0],
246 ptc.kspValues[ampName][i],
247 )
248 self.assertFloatsAlmostEqual(
249 extractPtc.covariances[ampName][0],
250 ptc.covariances[ampName][i],
251 )
252 self.assertFloatsAlmostEqual(
253 extractPtc.covariancesSqrtWeights[ampName][0],
254 ptc.covariancesSqrtWeights[ampName][i],
255 )
257 mask = ptc.getGoodPoints(amp)
259 values = ((ptc.covariancesModel[amp][mask, 0, 0] - ptc.covariances[amp][mask, 0, 0])
260 / ptc.covariancesModel[amp][mask, 0, 0])
261 np.testing.assert_array_less(np.abs(values), 2e-3)
263 values = ((ptc.covariancesModel[amp][mask, 1, 1] - ptc.covariances[amp][mask, 1, 1])
264 / ptc.covariancesModel[amp][mask, 1, 1])
265 np.testing.assert_array_less(np.abs(values), 0.2)
267 values = ((ptc.covariancesModel[amp][mask, 1, 2] - ptc.covariances[amp][mask, 1, 2])
268 / ptc.covariancesModel[amp][mask, 1, 2])
269 np.testing.assert_array_less(np.abs(values), 0.2)
271 expIdsUsed = ptc.getExpIdsUsed("C:0,0")
272 # Check that these are the same as the inputs, paired up, with the
273 # first two (low flux) and final four (outliers, nans) removed.
274 self.assertTrue(np.all(expIdsUsed == np.array(expIds).reshape(len(expIds) // 2, 2)[1:-2]))
276 goodAmps = ptc.getGoodAmps()
277 self.assertEqual(goodAmps, self.ampNames)
279 # Check that every possibly modified field has the same length.
280 covShape = None
281 covSqrtShape = None
282 covModelShape = None
283 covModelNoBShape = None
285 for ampName in self.ampNames:
286 if covShape is None:
287 covShape = ptc.covariances[ampName].shape
288 covSqrtShape = ptc.covariancesSqrtWeights[ampName].shape
289 covModelShape = ptc.covariancesModel[ampName].shape
290 covModelNoBShape = ptc.covariancesModelNoB[ampName].shape
291 else:
292 self.assertEqual(ptc.covariances[ampName].shape, covShape)
293 self.assertEqual(ptc.covariancesSqrtWeights[ampName].shape, covSqrtShape)
294 self.assertEqual(ptc.covariancesModel[ampName].shape, covModelShape)
295 self.assertEqual(ptc.covariancesModelNoB[ampName].shape, covModelNoBShape)
297 # And check that this is serializable
298 with tempfile.NamedTemporaryFile(suffix=".fits") as f:
299 usedFilename = ptc.writeFits(f.name)
300 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
301 self.assertEqual(fromFits, ptc)
303 def ptcFitAndCheckPtc(
304 self,
305 order=None,
306 fitType=None,
307 doTableArray=False,
308 doFitBootstrap=False,
309 doLegacy=False,
310 ):
311 localDataset = copy.deepcopy(self.dataset)
312 localDataset.ptcFitType = fitType
313 configSolve = copy.copy(self.defaultConfigSolve)
314 configLin = cpPipe.linearity.LinearitySolveTask.ConfigClass()
315 placesTests = 6
316 if doFitBootstrap:
317 configSolve.doFitBootstrap = True
318 # Bootstrap method in cp_pipe/utils.py does multiple fits
319 # in the precense of noise. Allow for more margin of
320 # error.
321 placesTests = 3
323 configSolve.doLegacyTurnoffSelection = doLegacy
325 if fitType == 'POLYNOMIAL':
326 if order not in [2, 3]:
327 RuntimeError("Enter a valid polynomial order for this test: 2 or 3")
328 if order == 2:
329 for ampName in self.ampNames:
330 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 for
331 mu in localDataset.rawMeans[ampName]]
332 configSolve.polynomialFitDegree = 2
333 if order == 3:
334 for ampName in self.ampNames:
335 localDataset.rawVars[ampName] = [self.noiseSq + self.c1*mu + self.c2*mu**2 + self.c3*mu**3
336 for mu in localDataset.rawMeans[ampName]]
337 configSolve.polynomialFitDegree = 3
338 elif fitType == 'EXPAPPROXIMATION':
339 g = self.gain
340 for ampName in self.ampNames:
341 localDataset.rawVars[ampName] = [(0.5/(self.a00*g**2)*(np.exp(2*self.a00*mu*g)-1)
342 + self.noiseSq/(g*g))
343 for mu in localDataset.rawMeans[ampName]]
344 else:
345 raise RuntimeError("Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'")
347 # Initialize mask and covariance weights that will be used in fits.
348 # Covariance weights values empirically determined from one of
349 # the cases in test_covAstier.
350 matrixSize = localDataset.covMatrixSide
351 maskLength = len(localDataset.rawMeans[ampName])
352 for ampName in self.ampNames:
353 localDataset.expIdMask[ampName] = np.repeat(True, maskLength)
354 localDataset.covariancesSqrtWeights[ampName] = np.repeat(np.ones((matrixSize, matrixSize)),
355 maskLength).reshape((maskLength,
356 matrixSize,
357 matrixSize))
358 localDataset.covariancesSqrtWeights[ampName][:, 0, 0] = [0.07980188, 0.01339653, 0.0073118,
359 0.00502802, 0.00383132, 0.00309475,
360 0.00259572, 0.00223528, 0.00196273,
361 0.00174943, 0.00157794, 0.00143707,
362 0.00131929, 0.00121935, 0.0011334,
363 0.00105893, 0.00099357, 0.0009358,
364 0.00088439, 0.00083833]
366 configLin.maxLookupTableAdu = 200000 # Max ADU in input mock flats
367 configLin.maxLinearAdu = 100000
368 configLin.minLinearAdu = 50000
369 if doTableArray:
370 configLin.linearityType = "LookupTable"
371 else:
372 configLin.linearityType = "Polynomial"
373 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=configSolve)
374 linearityTask = cpPipe.linearity.LinearitySolveTask(config=configLin)
376 if doTableArray:
377 # Non-linearity
378 numberAmps = len(self.ampNames)
379 # localDataset: PTC dataset
380 # (`lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`)
381 localDataset = solveTask.fitMeasurementsToModel(localDataset)
382 # linDataset here is a lsst.pipe.base.Struct
383 linDataset = linearityTask.run(localDataset,
384 dummy=[1.0],
385 camera=FakeCamera([self.flatExp1.getDetector()]),
386 inputPhotodiodeData={},
387 inputDims={'detector': 0})
388 linDataset = linDataset.outputLinearizer
389 else:
390 localDataset = solveTask.fitMeasurementsToModel(localDataset)
391 linDataset = linearityTask.run(localDataset,
392 dummy=[1.0],
393 camera=FakeCamera([self.flatExp1.getDetector()]),
394 inputPhotodiodeData={},
395 inputDims={'detector': 0})
396 linDataset = linDataset.outputLinearizer
397 if doTableArray:
398 # check that the linearizer table has been filled out properly
399 for i in np.arange(numberAmps):
400 tMax = (configLin.maxLookupTableAdu)/self.flux
401 timeRange = np.linspace(0., tMax, configLin.maxLookupTableAdu)
402 signalIdeal = timeRange*self.flux
403 signalUncorrected = funcPolynomial(np.array([0.0, self.flux, self.k2NonLinearity]),
404 timeRange)
405 linearizerTableRow = signalIdeal - signalUncorrected
406 self.assertEqual(len(linearizerTableRow), len(linDataset.tableData[i, :]))
407 for j in np.arange(len(linearizerTableRow)):
408 self.assertAlmostEqual(linearizerTableRow[j], linDataset.tableData[i, :][j],
409 places=placesTests)
410 else:
411 # check entries in localDataset, which was modified by the function
412 for ampName in self.ampNames:
413 maskAmp = localDataset.expIdMask[ampName]
414 finalMuVec = localDataset.rawMeans[ampName][maskAmp]
415 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp]
416 linearPart = self.flux*finalTimeVec
417 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart
418 self.assertEqual(fitType, localDataset.ptcFitType)
419 self.assertAlmostEqual(self.gain, localDataset.gain[ampName])
420 if fitType == 'POLYNOMIAL':
421 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1])
422 self.assertAlmostEqual(np.sqrt(self.noiseSq)*self.gain, localDataset.noise[ampName])
423 if fitType == 'EXPAPPROXIMATION':
424 self.assertAlmostEqual(self.a00, localDataset.ptcFitPars[ampName][0])
425 # noise already in electrons for 'EXPAPPROXIMATION' fit
426 self.assertAlmostEqual(np.sqrt(self.noiseSq), localDataset.noise[ampName])
428 # check entries in returned dataset (a dict of , for nonlinearity)
429 for ampName in self.ampNames:
430 maskAmp = localDataset.expIdMask[ampName]
431 finalMuVec = localDataset.rawMeans[ampName][maskAmp]
432 finalTimeVec = localDataset.rawExpTimes[ampName][maskAmp]
433 linearPart = self.flux*finalTimeVec
434 inputFracNonLinearityResiduals = 100*(linearPart - finalMuVec)/linearPart
436 # Nonlinearity fit parameters
437 # Polynomial fits are now normalized to unit flux scaling
438 self.assertAlmostEqual(0.0, linDataset.fitParams[ampName][0], places=1)
439 self.assertAlmostEqual(1.0, linDataset.fitParams[ampName][1],
440 places=5)
442 # Non-linearity coefficient for linearizer
443 squaredCoeff = self.k2NonLinearity/(self.flux**2)
444 self.assertAlmostEqual(squaredCoeff, linDataset.fitParams[ampName][2],
445 places=placesTests)
446 self.assertAlmostEqual(-squaredCoeff, linDataset.linearityCoeffs[ampName][2],
447 places=placesTests)
449 linearPartModel = linDataset.fitParams[ampName][1]*finalTimeVec*self.flux
450 outputFracNonLinearityResiduals = 100*(linearPartModel - finalMuVec)/linearPartModel
451 # Fractional nonlinearity residuals
452 self.assertEqual(len(outputFracNonLinearityResiduals), len(inputFracNonLinearityResiduals))
453 for calc, truth in zip(outputFracNonLinearityResiduals, inputFracNonLinearityResiduals):
454 self.assertAlmostEqual(calc, truth, places=3)
456 def test_ptcFit(self):
457 for createArray in [True, False]:
458 for doLegacy in [False, True]:
459 for (fitType, order) in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('EXPAPPROXIMATION', None)]:
460 self.ptcFitAndCheckPtc(
461 fitType=fitType,
462 order=order,
463 doTableArray=createArray,
464 doLegacy=doLegacy,
465 )
467 def test_meanVarMeasurement(self):
468 task = self.defaultTaskExtract
469 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(self.flatExp1,
470 self.flatExp2)
471 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
473 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1)
474 self.assertLess(self.flatMean - mu, 1)
476 def test_meanVarMeasurementWithNans(self):
477 task = self.defaultTaskExtract
479 flatExp1 = self.flatExp1.clone()
480 flatExp2 = self.flatExp2.clone()
482 flatExp1.image.array[20:30, :] = np.nan
483 flatExp2.image.array[20:30, :] = np.nan
485 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(flatExp1,
486 flatExp2)
487 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
489 expectedMu1 = np.nanmean(flatExp1.image.array)
490 expectedMu2 = np.nanmean(flatExp2.image.array)
491 expectedMu = 0.5*(expectedMu1 + expectedMu2)
493 # Now the variance of the difference. First, create the diff image.
494 im1 = flatExp1.maskedImage
495 im2 = flatExp2.maskedImage
497 temp = im2.clone()
498 temp *= expectedMu1
499 diffIm = im1.clone()
500 diffIm *= expectedMu2
501 diffIm -= temp
502 diffIm /= expectedMu
504 # Divide by two as it is what measureMeanVarCov returns
505 # (variance of difference)
506 expectedVar = 0.5*np.nanvar(diffIm.image.array)
508 # Check that the standard deviations and the emans agree to
509 # less than 1 ADU
510 self.assertLess(np.sqrt(expectedVar) - np.sqrt(varDiff), 1)
511 self.assertLess(expectedMu - mu, 1)
513 def test_meanVarMeasurementAllNan(self):
514 task = self.defaultTaskExtract
515 flatExp1 = self.flatExp1.clone()
516 flatExp2 = self.flatExp2.clone()
518 flatExp1.image.array[:, :] = np.nan
519 flatExp2.image.array[:, :] = np.nan
521 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(flatExp1,
522 flatExp2)
523 mu, varDiff, covDiff = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
525 self.assertTrue(np.isnan(mu))
526 self.assertTrue(np.isnan(varDiff))
527 self.assertTrue(covDiff is None)
529 def test_meanVarMeasurementTooFewPixels(self):
530 task = self.defaultTaskExtract
531 flatExp1 = self.flatExp1.clone()
532 flatExp2 = self.flatExp2.clone()
534 flatExp1.image.array[0: 190, :] = np.nan
535 flatExp2.image.array[0: 190, :] = np.nan
537 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"]
538 flatExp1.mask.array[0: 190, :] &= 2**bit
539 flatExp2.mask.array[0: 190, :] &= 2**bit
541 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(flatExp1,
542 flatExp2)
543 with self.assertLogs(level=logging.WARNING) as cm:
544 mu, varDiff, covDiff = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
545 self.assertIn("Number of good points", cm.output[0])
547 self.assertTrue(np.isnan(mu))
548 self.assertTrue(np.isnan(varDiff))
549 self.assertTrue(covDiff is None)
551 def test_meanVarMeasurementTooNarrowStrip(self):
552 # We need a new config to make sure the second covariance cut is
553 # triggered.
554 config = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
555 config.minNumberGoodPixelsForCovariance = 10
556 task = cpPipe.ptc.PhotonTransferCurveExtractTask(config=config)
557 flatExp1 = self.flatExp1.clone()
558 flatExp2 = self.flatExp2.clone()
560 flatExp1.image.array[0: 195, :] = np.nan
561 flatExp2.image.array[0: 195, :] = np.nan
562 flatExp1.image.array[:, 0: 195] = np.nan
563 flatExp2.image.array[:, 0: 195] = np.nan
565 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"]
566 flatExp1.mask.array[0: 195, :] &= 2**bit
567 flatExp2.mask.array[0: 195, :] &= 2**bit
568 flatExp1.mask.array[:, 0: 195] &= 2**bit
569 flatExp2.mask.array[:, 0: 195] &= 2**bit
571 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(flatExp1,
572 flatExp2)
573 with self.assertLogs(level=logging.WARNING) as cm:
574 mu, varDiff, covDiff = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
575 self.assertIn("Not enough pixels", cm.output[0])
577 self.assertTrue(np.isnan(mu))
578 self.assertTrue(np.isnan(varDiff))
579 self.assertTrue(covDiff is None)
581 def test_makeZeroSafe(self):
582 noZerosArray = [1., 20, -35, 45578.98, 90.0, 897, 659.8]
583 someZerosArray = [1., 20, 0, 0, 90, 879, 0]
584 allZerosArray = [0., 0.0, 0, 0, 0.0, 0, 0]
586 substituteValue = 1e-10
588 expectedSomeZerosArray = [1., 20, substituteValue, substituteValue, 90, 879, substituteValue]
589 expectedAllZerosArray = np.repeat(substituteValue, len(allZerosArray))
591 measuredSomeZerosArray = self.defaultTaskSolve._makeZeroSafe(someZerosArray,
592 substituteValue=substituteValue)
593 measuredAllZerosArray = self.defaultTaskSolve._makeZeroSafe(allZerosArray,
594 substituteValue=substituteValue)
595 measuredNoZerosArray = self.defaultTaskSolve._makeZeroSafe(noZerosArray,
596 substituteValue=substituteValue)
598 for exp, meas in zip(expectedSomeZerosArray, measuredSomeZerosArray):
599 self.assertEqual(exp, meas)
600 for exp, meas in zip(expectedAllZerosArray, measuredAllZerosArray):
601 self.assertEqual(exp, meas)
602 for exp, meas in zip(noZerosArray, measuredNoZerosArray):
603 self.assertEqual(exp, meas)
605 def test_getInitialGoodPoints(self):
606 xs = [1, 2, 3, 4, 5, 6]
607 ys = [2*x for x in xs]
608 points = self.defaultTaskSolve._getInitialGoodPoints(xs, ys, minVarPivotSearch=0.,
609 consecutivePointsVarDecreases=2)
610 assert np.all(points) == np.all(np.array([True for x in xs]))
612 ys[4] = 7 # Variance decreases in two consecutive points after ys[3]=8
613 ys[5] = 6
614 points = self.defaultTaskSolve._getInitialGoodPoints(xs, ys, minVarPivotSearch=0.,
615 consecutivePointsVarDecreases=2)
616 assert np.all(points) == np.all(np.array([True, True, True, True, False]))
618 def runGetGainFromFlatPair(self, correctionType='NONE'):
619 extractConfig = self.defaultConfigExtract
620 extractConfig.gainCorrectionType = correctionType
621 extractConfig.minNumberGoodPixelsForCovariance = 5000
622 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig)
624 expDict = {}
625 expIds = []
626 idCounter = 0
627 inputGain = self.gain # 1.5 e/ADU
628 for expTime in self.timeVec:
629 # Approximation works better at low flux, e.g., < 10000 ADU
630 mockExp1, mockExp2 = makeMockFlats(expTime, gain=inputGain,
631 readNoiseElectrons=np.sqrt(self.noiseSq),
632 fluxElectrons=100,
633 expId1=idCounter, expId2=idCounter+1)
634 mockExpRef1 = PretendRef(mockExp1)
635 mockExpRef2 = PretendRef(mockExp2)
636 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter+1))
637 expIds.append(idCounter)
638 expIds.append(idCounter+1)
639 idCounter += 2
641 resultsExtract = extractTask.run(inputExp=expDict, inputDims=expIds,
642 taskMetadata=[self.metadataContents for x in expIds])
643 for exposurePair in resultsExtract.outputCovariances:
644 for ampName in self.ampNames:
645 if exposurePair.gain[ampName] is np.nan:
646 continue
647 self.assertAlmostEqual(exposurePair.gain[ampName], inputGain, delta=0.04)
649 def test_getGainFromFlatPair(self):
650 for gainCorrectionType in ['NONE', 'SIMPLE', 'FULL', ]:
651 self.runGetGainFromFlatPair(gainCorrectionType)
654class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase):
655 def setUp(self):
656 self.ptcData = PhotonTransferCurveDataset(['C00', 'C01'], " ")
657 self.ptcData.inputExpIdPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
658 'C01': [(123, 234), (345, 456), (567, 678)]}
660 def test_generalBehaviour(self):
661 test = PhotonTransferCurveDataset(['C00', 'C01'], " ")
662 test.inputExpIdPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
663 'C01': [(123, 234), (345, 456), (567, 678)]}
666class TestMemory(lsst.utils.tests.MemoryTestCase):
667 pass
670def setup_module(module):
671 lsst.utils.tests.init()
674if __name__ == "__main__": 674 ↛ 675line 674 didn't jump to line 675, because the condition on line 674 was never true
675 lsst.utils.tests.init()
676 unittest.main()