Coverage for tests/test_ptc.py: 9%
368 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-09-02 11:08 +0000
« prev ^ index » next coverage.py v7.3.0, created at 2023-09-02 11:08 +0000
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe."""
27import unittest
28import numpy as np
29import copy
30import tempfile
31import logging
33import lsst.utils
34import lsst.utils.tests
36import lsst.cp.pipe as cpPipe
37import lsst.ip.isr.isrMock as isrMock
38from lsst.ip.isr import PhotonTransferCurveDataset, PhotodiodeCalib
39from lsst.cp.pipe.utils import makeMockFlats
41from lsst.pipe.base import InMemoryDatasetHandle, TaskMetadata
44class FakeCamera(list):
45 def getName(self):
46 return "FakeCam"
49class PretendRef:
50 "A class to act as a mock exposure reference"
52 def __init__(self, exposure):
53 self.exp = exposure
55 def get(self, component=None):
56 if component == "visitInfo":
57 return self.exp.getVisitInfo()
58 elif component == "detector":
59 return self.exp.getDetector()
60 elif component == "metadata":
61 return self.exp.getMetadata()
62 else:
63 return self.exp
66class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase):
67 """A test case for the PTC tasks."""
69 def setUp(self):
70 self.defaultConfigExtract = (
71 cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
72 )
73 self.defaultTaskExtract = cpPipe.ptc.PhotonTransferCurveExtractTask(
74 config=self.defaultConfigExtract
75 )
77 self.defaultConfigSolve = cpPipe.ptc.PhotonTransferCurveSolveTask.ConfigClass()
78 self.defaultTaskSolve = cpPipe.ptc.PhotonTransferCurveSolveTask(
79 config=self.defaultConfigSolve
80 )
82 self.flatMean = 2000
83 self.readNoiseAdu = 10
84 mockImageConfig = isrMock.IsrMock.ConfigClass()
86 # flatDrop is not really relevant as we replace the data
87 # but good to note it in case we change how this image is made
88 mockImageConfig.flatDrop = 0.99999
89 mockImageConfig.isTrimmed = True
91 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
92 self.flatExp2 = self.flatExp1.clone()
93 (shapeY, shapeX) = self.flatExp1.getDimensions()
95 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
97 self.rng1 = np.random.RandomState(1984)
98 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
99 self.rng2 = np.random.RandomState(666)
100 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
102 self.flatExp1.image.array[:] = flatData1
103 self.flatExp2.image.array[:] = flatData2
105 # create fake PTC data to see if fit works, for one amp ('amp')
106 self.flux = 1000.0 # ADU/sec
107 self.timeVec = np.arange(1.0, 101.0, 5)
108 self.k2NonLinearity = -5e-6
109 # quadratic signal-chain non-linearity
110 muVec = self.flux * self.timeVec + self.k2NonLinearity * self.timeVec**2
111 self.gain = 0.75 # e-/ADU
112 self.c1 = 1.0 / self.gain
113 self.noiseSq = 2 * self.gain # 7.5 (e-)^2
114 self.a00 = -1.2e-6
115 self.c2 = -1.5e-6
116 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean
117 self.photoCharges = np.linspace(1e-8, 1e-5, len(self.timeVec))
119 self.ampNames = [
120 amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()
121 ]
122 self.dataset = PhotonTransferCurveDataset(self.ampNames, ptcFitType="PARTIAL")
123 self.covariancesSqrtWeights = {}
124 for (
125 ampName
126 ) in self.ampNames: # just the expTimes and means here - vars vary per function
127 self.dataset.rawExpTimes[ampName] = self.timeVec
128 self.dataset.rawMeans[ampName] = muVec
129 self.dataset.covariancesSqrtWeights[ampName] = np.zeros(
130 (1, self.dataset.covMatrixSide, self.dataset.covMatrixSide)
131 )
133 # ISR metadata
134 self.metadataContents = TaskMetadata()
135 self.metadataContents["isr"] = {}
136 # Overscan readout noise [in ADU]
137 for amp in self.ampNames:
138 self.metadataContents["isr"][f"RESIDUAL STDEV {amp}"] = (
139 np.sqrt(self.noiseSq) / self.gain
140 )
142 def test_covAstier(self):
143 """Test to check getCovariancesAstier
145 We check that the gain is the same as the imput gain from the
146 mock data, that the covariances via FFT (as it is in
147 MeasurePhotonTransferCurveTask when doCovariancesAstier=True)
148 are the same as calculated in real space, and that Cov[0, 0]
149 (i.e., the variances) are similar to the variances calculated
150 with the standard method (when doCovariancesAstier=false),
152 """
153 extractConfig = self.defaultConfigExtract
154 extractConfig.minNumberGoodPixelsForCovariance = 5000
155 extractConfig.detectorMeasurementRegion = "FULL"
156 extractConfig.doExtractPhotodiodeData = True
157 extractConfig.auxiliaryHeaderKeys = ["CCOBCURR", "CCDTEMP"]
158 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig)
160 solveConfig = self.defaultConfigSolve
161 solveConfig.ptcFitType = "FULLCOVARIANCE"
162 # Cut off the low-flux point which is a bad fit, and this
163 # also exercises this functionality and makes the tests
164 # run a lot faster.
165 solveConfig.minMeanSignal["ALL_AMPS"] = 2000.0
166 # Set the outlier fit threshold higher than the default appropriate
167 # for this test dataset.
168 solveConfig.maxSignalInitialPtcOutlierFit = 90000.0
169 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=solveConfig)
171 inputGain = self.gain
173 muStandard, varStandard = {}, {}
174 expDict = {}
175 expIds = []
176 pdHandles = []
177 idCounter = 0
178 for i, expTime in enumerate(self.timeVec):
179 mockExp1, mockExp2 = makeMockFlats(
180 expTime,
181 gain=inputGain,
182 readNoiseElectrons=3,
183 expId1=idCounter,
184 expId2=idCounter + 1,
185 )
186 for mockExp in [mockExp1, mockExp2]:
187 md = mockExp.getMetadata()
188 # These values are chosen to be easily compared after
189 # processing for correct ordering.
190 md['CCOBCURR'] = float(idCounter)
191 md['CCDTEMP'] = float(idCounter + 1)
192 mockExp.setMetadata(md)
194 mockExpRef1 = PretendRef(mockExp1)
195 mockExpRef2 = PretendRef(mockExp2)
196 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1))
197 expIds.append(idCounter)
198 expIds.append(idCounter + 1)
199 for ampNumber, ampName in enumerate(self.ampNames):
200 # cov has (i, j, var, cov, npix)
201 (
202 im1Area,
203 im2Area,
204 imStatsCtrl,
205 mu1,
206 mu2,
207 ) = extractTask.getImageAreasMasksStats(mockExp1, mockExp2)
208 muDiff, varDiff, covAstier = extractTask.measureMeanVarCov(
209 im1Area, im2Area, imStatsCtrl, mu1, mu2
210 )
211 muStandard.setdefault(ampName, []).append(muDiff)
212 varStandard.setdefault(ampName, []).append(varDiff)
214 # Make a photodiode dataset to integrate.
215 timeSamples = np.linspace(0, 20.0, 100)
216 currentSamples = np.zeros(100)
217 currentSamples[50] = -1.0*self.photoCharges[i]
219 pdCalib = PhotodiodeCalib(timeSamples=timeSamples, currentSamples=currentSamples)
220 pdCalib.currentScale = -1.0
221 pdCalib.integrationMethod = "CHARGE_SUM"
223 pdHandles.append(
224 InMemoryDatasetHandle(
225 pdCalib,
226 dataId={"exposure": idCounter},
227 )
228 )
229 pdHandles.append(
230 InMemoryDatasetHandle(
231 pdCalib,
232 dataId={"exposure": idCounter + 1},
233 )
234 )
235 idCounter += 2
237 resultsExtract = extractTask.run(
238 inputExp=expDict,
239 inputDims=expIds,
240 taskMetadata=[self.metadataContents for x in expIds],
241 inputPhotodiodeData=pdHandles,
242 )
244 # Force the last PTC dataset to have a NaN, and ensure that the
245 # task runs (DM-38029). This is a minor perturbation and does not
246 # affect the output comparison. Note that we use index -2 because
247 # these datasets are in pairs of [real, dummy] to match the inputs
248 # to the extract task.
249 resultsExtract.outputCovariances[-2].rawMeans["C:0,0"] = np.array([np.nan])
250 resultsExtract.outputCovariances[-2].rawVars["C:0,0"] = np.array([np.nan])
252 # Force the next-to-last PTC dataset to have a decreased variance to
253 # ensure that the outlier fit rejection works. Note that we use
254 # index -4 because these datasets are in pairs of [real, dummy] to
255 # match the inputs to the extract task.
256 rawVar = resultsExtract.outputCovariances[-4].rawVars["C:0,0"]
257 resultsExtract.outputCovariances[-4].rawVars["C:0,0"] = rawVar * 0.9
259 # Reorganize the outputCovariances so we can confirm they come
260 # out sorted afterwards.
261 outputCovariancesRev = resultsExtract.outputCovariances[::-1]
263 resultsSolve = solveTask.run(
264 outputCovariancesRev, camera=FakeCamera([self.flatExp1.getDetector()])
265 )
267 ptc = resultsSolve.outputPtcDataset
269 # Some expected values for noise matrix, just to check that
270 # it was calculated.
271 noiseMatrixNoBExpected = {
272 (0, 0): 6.53126505,
273 (1, 1): -23.20924747,
274 (2, 2): 35.69834113,
275 }
276 noiseMatrixExpected = {
277 (0, 0): 29.37146918,
278 (1, 1): -14.6849025,
279 (2, 2): 24.7328517,
280 }
282 noiseMatrixExpected = np.array(
283 [
284 [
285 29.37146918,
286 9.2760363,
287 -29.08907932,
288 33.65818827,
289 -52.65710984,
290 -18.5821773,
291 -46.26896286,
292 65.01049736,
293 ],
294 [
295 -3.62427987,
296 -14.6849025,
297 -46.55230305,
298 -1.30410627,
299 6.44903599,
300 18.11796075,
301 -22.72874074,
302 20.90219857,
303 ],
304 [
305 5.09203058,
306 -4.40097862,
307 24.7328517,
308 39.2847586,
309 -21.46132351,
310 8.12179783,
311 6.23585617,
312 -2.09949622,
313 ],
314 [
315 35.79204016,
316 -6.50205005,
317 3.37910363,
318 15.22335662,
319 -19.29035067,
320 9.66065941,
321 7.47510934,
322 20.25962845,
323 ],
324 [
325 -36.23187633,
326 -22.72307472,
327 16.29140749,
328 -13.09493835,
329 3.32091085,
330 52.4380977,
331 -8.06428902,
332 -22.66669839,
333 ],
334 [
335 -27.93122896,
336 15.37016686,
337 9.18835073,
338 -24.48892946,
339 8.14480304,
340 22.38983222,
341 22.36866891,
342 -0.38803439,
343 ],
344 [
345 17.13962665,
346 -28.33153763,
347 -17.79744334,
348 -18.57064463,
349 7.69408833,
350 8.48265396,
351 18.0447022,
352 -16.97496022,
353 ],
354 [
355 10.09078383,
356 -26.61613002,
357 10.48504889,
358 15.33196998,
359 -23.35165517,
360 -24.53098643,
361 -18.21201067,
362 17.40755051,
363 ],
364 ]
365 )
367 noiseMatrixNoBExpected = np.array(
368 [
369 [
370 6.53126505,
371 12.14827594,
372 -37.11919923,
373 41.18675353,
374 -85.1613845,
375 -28.45801954,
376 -61.24442999,
377 88.76480122,
378 ],
379 [
380 -4.64541165,
381 -23.20924747,
382 -66.08733987,
383 -0.87558055,
384 12.20111853,
385 24.84795549,
386 -34.92458788,
387 24.42745014,
388 ],
389 [
390 7.66734507,
391 -4.51403645,
392 35.69834113,
393 52.73693356,
394 -30.85044089,
395 10.86761771,
396 10.8503068,
397 -2.18908327,
398 ],
399 [
400 50.9901156,
401 -7.34803977,
402 5.33443765,
403 21.60899396,
404 -25.06129827,
405 15.14015505,
406 10.94263771,
407 29.23975515,
408 ],
409 [
410 -48.66912069,
411 -31.58003774,
412 21.81305735,
413 -13.08993444,
414 8.17275394,
415 74.85293723,
416 -11.18403252,
417 -31.7799437,
418 ],
419 [
420 -38.55206382,
421 22.92982676,
422 13.39861008,
423 -33.3307362,
424 8.65362238,
425 29.18775548,
426 31.78433947,
427 1.27923706,
428 ],
429 [
430 23.33663918,
431 -41.74105625,
432 -26.55920751,
433 -24.71611677,
434 12.13343146,
435 11.25763907,
436 21.79131019,
437 -26.579393,
438 ],
439 [
440 11.44334226,
441 -34.9759641,
442 13.96449509,
443 19.64121933,
444 -36.09794843,
445 -34.27205933,
446 -25.16574105,
447 23.80460972,
448 ],
449 ]
450 )
452 for amp in self.ampNames:
453 self.assertAlmostEqual(ptc.gain[amp], inputGain, places=2)
454 for v1, v2 in zip(varStandard[amp], ptc.finalVars[amp]):
455 self.assertAlmostEqual(v1 / v2, 1.0, places=1)
457 # Check that the PTC turnoff is correctly computed.
458 # This will be different for the C:0,0 amp.
459 if amp == "C:0,0":
460 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-3])
461 else:
462 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-1])
464 # Test that all the quantities are correctly ordered and have
465 # not accidentally been masked. We check every other output ([::2])
466 # because these datasets are in pairs of [real, dummy] to
467 # match the inputs to the extract task.
468 for i, extractPtc in enumerate(resultsExtract.outputCovariances[::2]):
469 self.assertFloatsAlmostEqual(
470 extractPtc.rawExpTimes[ampName][0],
471 ptc.rawExpTimes[ampName][i],
472 )
473 self.assertFloatsAlmostEqual(
474 extractPtc.rawMeans[ampName][0],
475 ptc.rawMeans[ampName][i],
476 )
477 self.assertFloatsAlmostEqual(
478 extractPtc.rawVars[ampName][0],
479 ptc.rawVars[ampName][i],
480 )
481 self.assertFloatsAlmostEqual(
482 extractPtc.photoCharges[ampName][0],
483 ptc.photoCharges[ampName][i],
484 )
485 self.assertFloatsAlmostEqual(
486 extractPtc.histVars[ampName][0],
487 ptc.histVars[ampName][i],
488 )
489 self.assertFloatsAlmostEqual(
490 extractPtc.histChi2Dofs[ampName][0],
491 ptc.histChi2Dofs[ampName][i],
492 )
493 self.assertFloatsAlmostEqual(
494 extractPtc.kspValues[ampName][0],
495 ptc.kspValues[ampName][i],
496 )
497 self.assertFloatsAlmostEqual(
498 extractPtc.covariances[ampName][0],
499 ptc.covariances[ampName][i],
500 )
501 self.assertFloatsAlmostEqual(
502 extractPtc.covariancesSqrtWeights[ampName][0],
503 ptc.covariancesSqrtWeights[ampName][i],
504 )
505 self.assertFloatsAlmostEqual(
506 ptc.noiseMatrix[ampName], noiseMatrixExpected, atol=1e-8, rtol=None
507 )
508 self.assertFloatsAlmostEqual(
509 ptc.noiseMatrixNoB[ampName],
510 noiseMatrixNoBExpected,
511 atol=1e-8,
512 rtol=None,
513 )
515 mask = ptc.getGoodPoints(amp)
517 values = (
518 ptc.covariancesModel[amp][mask, 0, 0] - ptc.covariances[amp][mask, 0, 0]
519 ) / ptc.covariancesModel[amp][mask, 0, 0]
520 np.testing.assert_array_less(np.abs(values), 2e-3)
522 values = (
523 ptc.covariancesModel[amp][mask, 1, 1] - ptc.covariances[amp][mask, 1, 1]
524 ) / ptc.covariancesModel[amp][mask, 1, 1]
525 np.testing.assert_array_less(np.abs(values), 0.2)
527 values = (
528 ptc.covariancesModel[amp][mask, 1, 2] - ptc.covariances[amp][mask, 1, 2]
529 ) / ptc.covariancesModel[amp][mask, 1, 2]
530 np.testing.assert_array_less(np.abs(values), 0.2)
532 # And test that the auxiliary values are there and correctly ordered.
533 self.assertIn('CCOBCURR', ptc.auxValues)
534 self.assertIn('CCDTEMP', ptc.auxValues)
535 firstExpIds = np.array([i for i, _ in ptc.inputExpIdPairs['C:0,0']], dtype=np.float64)
536 self.assertFloatsAlmostEqual(ptc.auxValues['CCOBCURR'], firstExpIds)
537 self.assertFloatsAlmostEqual(ptc.auxValues['CCDTEMP'], firstExpIds + 1)
539 expIdsUsed = ptc.getExpIdsUsed("C:0,0")
540 # Check that these are the same as the inputs, paired up, with the
541 # first two (low flux) and final four (outliers, nans) removed.
542 self.assertTrue(
543 np.all(expIdsUsed == np.array(expIds).reshape(len(expIds) // 2, 2)[1:-2])
544 )
546 goodAmps = ptc.getGoodAmps()
547 self.assertEqual(goodAmps, self.ampNames)
549 # Check that every possibly modified field has the same length.
550 covShape = None
551 covSqrtShape = None
552 covModelShape = None
553 covModelNoBShape = None
555 for ampName in self.ampNames:
556 if covShape is None:
557 covShape = ptc.covariances[ampName].shape
558 covSqrtShape = ptc.covariancesSqrtWeights[ampName].shape
559 covModelShape = ptc.covariancesModel[ampName].shape
560 covModelNoBShape = ptc.covariancesModelNoB[ampName].shape
561 else:
562 self.assertEqual(ptc.covariances[ampName].shape, covShape)
563 self.assertEqual(
564 ptc.covariancesSqrtWeights[ampName].shape, covSqrtShape
565 )
566 self.assertEqual(ptc.covariancesModel[ampName].shape, covModelShape)
567 self.assertEqual(
568 ptc.covariancesModelNoB[ampName].shape, covModelNoBShape
569 )
571 # And check that this is serializable
572 with tempfile.NamedTemporaryFile(suffix=".fits") as f:
573 usedFilename = ptc.writeFits(f.name)
574 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
575 self.assertEqual(fromFits, ptc)
577 def ptcFitAndCheckPtc(
578 self,
579 order=None,
580 fitType=None,
581 doFitBootstrap=False,
582 doLegacy=False,
583 ):
584 localDataset = copy.deepcopy(self.dataset)
585 localDataset.ptcFitType = fitType
586 configSolve = copy.copy(self.defaultConfigSolve)
587 if doFitBootstrap:
588 configSolve.doFitBootstrap = True
590 configSolve.doLegacyTurnoffSelection = doLegacy
592 if fitType == "POLYNOMIAL":
593 if order not in [2, 3]:
594 RuntimeError("Enter a valid polynomial order for this test: 2 or 3")
595 if order == 2:
596 for ampName in self.ampNames:
597 localDataset.rawVars[ampName] = [
598 self.noiseSq + self.c1 * mu + self.c2 * mu**2
599 for mu in localDataset.rawMeans[ampName]
600 ]
601 configSolve.polynomialFitDegree = 2
602 if order == 3:
603 for ampName in self.ampNames:
604 localDataset.rawVars[ampName] = [
605 self.noiseSq
606 + self.c1 * mu
607 + self.c2 * mu**2
608 + self.c3 * mu**3
609 for mu in localDataset.rawMeans[ampName]
610 ]
611 configSolve.polynomialFitDegree = 3
612 elif fitType == "EXPAPPROXIMATION":
613 g = self.gain
614 for ampName in self.ampNames:
615 localDataset.rawVars[ampName] = [
616 (
617 0.5 / (self.a00 * g**2) * (np.exp(2 * self.a00 * mu * g) - 1)
618 + self.noiseSq / (g * g)
619 )
620 for mu in localDataset.rawMeans[ampName]
621 ]
622 else:
623 raise RuntimeError(
624 "Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'"
625 )
627 # Initialize mask and covariance weights that will be used in fits.
628 # Covariance weights values empirically determined from one of
629 # the cases in test_covAstier.
630 matrixSize = localDataset.covMatrixSide
631 maskLength = len(localDataset.rawMeans[ampName])
632 for ampName in self.ampNames:
633 localDataset.expIdMask[ampName] = np.repeat(True, maskLength)
634 localDataset.covariancesSqrtWeights[ampName] = np.repeat(
635 np.ones((matrixSize, matrixSize)), maskLength
636 ).reshape((maskLength, matrixSize, matrixSize))
637 localDataset.covariancesSqrtWeights[ampName][:, 0, 0] = [
638 0.07980188,
639 0.01339653,
640 0.0073118,
641 0.00502802,
642 0.00383132,
643 0.00309475,
644 0.00259572,
645 0.00223528,
646 0.00196273,
647 0.00174943,
648 0.00157794,
649 0.00143707,
650 0.00131929,
651 0.00121935,
652 0.0011334,
653 0.00105893,
654 0.00099357,
655 0.0009358,
656 0.00088439,
657 0.00083833,
658 ]
660 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=configSolve)
662 localDataset = solveTask.fitMeasurementsToModel(localDataset)
664 # check entries in localDataset, which was modified by the function
665 for ampName in self.ampNames:
666 self.assertEqual(fitType, localDataset.ptcFitType)
667 self.assertAlmostEqual(self.gain, localDataset.gain[ampName])
668 if fitType == "POLYNOMIAL":
669 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1])
670 self.assertAlmostEqual(
671 np.sqrt(self.noiseSq) * self.gain, localDataset.noise[ampName]
672 )
673 if fitType == "EXPAPPROXIMATION":
674 self.assertAlmostEqual(
675 self.a00, localDataset.ptcFitPars[ampName][0]
676 )
677 # noise already in electrons for 'EXPAPPROXIMATION' fit
678 self.assertAlmostEqual(
679 np.sqrt(self.noiseSq), localDataset.noise[ampName]
680 )
682 def test_ptcFit(self):
683 for doLegacy in [False, True]:
684 for fitType, order in [
685 ("POLYNOMIAL", 2),
686 ("POLYNOMIAL", 3),
687 ("EXPAPPROXIMATION", None),
688 ]:
689 self.ptcFitAndCheckPtc(
690 fitType=fitType,
691 order=order,
692 doLegacy=doLegacy,
693 )
695 def test_meanVarMeasurement(self):
696 task = self.defaultTaskExtract
697 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
698 self.flatExp1, self.flatExp2
699 )
700 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
702 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1)
703 self.assertLess(self.flatMean - mu, 1)
705 def test_meanVarMeasurementWithNans(self):
706 task = self.defaultTaskExtract
708 flatExp1 = self.flatExp1.clone()
709 flatExp2 = self.flatExp2.clone()
711 flatExp1.image.array[20:30, :] = np.nan
712 flatExp2.image.array[20:30, :] = np.nan
714 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
715 flatExp1, flatExp2
716 )
717 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
719 expectedMu1 = np.nanmean(flatExp1.image.array)
720 expectedMu2 = np.nanmean(flatExp2.image.array)
721 expectedMu = 0.5 * (expectedMu1 + expectedMu2)
723 # Now the variance of the difference. First, create the diff image.
724 im1 = flatExp1.maskedImage
725 im2 = flatExp2.maskedImage
727 temp = im2.clone()
728 temp *= expectedMu1
729 diffIm = im1.clone()
730 diffIm *= expectedMu2
731 diffIm -= temp
732 diffIm /= expectedMu
734 # Divide by two as it is what measureMeanVarCov returns
735 # (variance of difference)
736 expectedVar = 0.5 * np.nanvar(diffIm.image.array)
738 # Check that the standard deviations and the emans agree to
739 # less than 1 ADU
740 self.assertLess(np.sqrt(expectedVar) - np.sqrt(varDiff), 1)
741 self.assertLess(expectedMu - mu, 1)
743 def test_meanVarMeasurementAllNan(self):
744 task = self.defaultTaskExtract
745 flatExp1 = self.flatExp1.clone()
746 flatExp2 = self.flatExp2.clone()
748 flatExp1.image.array[:, :] = np.nan
749 flatExp2.image.array[:, :] = np.nan
751 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
752 flatExp1, flatExp2
753 )
754 mu, varDiff, covDiff = task.measureMeanVarCov(
755 im1Area, im2Area, imStatsCtrl, mu1, mu2
756 )
758 self.assertTrue(np.isnan(mu))
759 self.assertTrue(np.isnan(varDiff))
760 self.assertTrue(covDiff is None)
762 def test_meanVarMeasurementTooFewPixels(self):
763 task = self.defaultTaskExtract
764 flatExp1 = self.flatExp1.clone()
765 flatExp2 = self.flatExp2.clone()
767 flatExp1.image.array[0:190, :] = np.nan
768 flatExp2.image.array[0:190, :] = np.nan
770 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"]
771 flatExp1.mask.array[0:190, :] &= 2**bit
772 flatExp2.mask.array[0:190, :] &= 2**bit
774 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
775 flatExp1, flatExp2
776 )
777 with self.assertLogs(level=logging.WARNING) as cm:
778 mu, varDiff, covDiff = task.measureMeanVarCov(
779 im1Area, im2Area, imStatsCtrl, mu1, mu2
780 )
781 self.assertIn("Number of good points", cm.output[0])
783 self.assertTrue(np.isnan(mu))
784 self.assertTrue(np.isnan(varDiff))
785 self.assertTrue(covDiff is None)
787 def test_meanVarMeasurementTooNarrowStrip(self):
788 # We need a new config to make sure the second covariance cut is
789 # triggered.
790 config = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
791 config.minNumberGoodPixelsForCovariance = 10
792 task = cpPipe.ptc.PhotonTransferCurveExtractTask(config=config)
793 flatExp1 = self.flatExp1.clone()
794 flatExp2 = self.flatExp2.clone()
796 flatExp1.image.array[0:195, :] = np.nan
797 flatExp2.image.array[0:195, :] = np.nan
798 flatExp1.image.array[:, 0:195] = np.nan
799 flatExp2.image.array[:, 0:195] = np.nan
801 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"]
802 flatExp1.mask.array[0:195, :] &= 2**bit
803 flatExp2.mask.array[0:195, :] &= 2**bit
804 flatExp1.mask.array[:, 0:195] &= 2**bit
805 flatExp2.mask.array[:, 0:195] &= 2**bit
807 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
808 flatExp1, flatExp2
809 )
810 with self.assertLogs(level=logging.WARNING) as cm:
811 mu, varDiff, covDiff = task.measureMeanVarCov(
812 im1Area, im2Area, imStatsCtrl, mu1, mu2
813 )
814 self.assertIn("Not enough pixels", cm.output[0])
816 self.assertTrue(np.isnan(mu))
817 self.assertTrue(np.isnan(varDiff))
818 self.assertTrue(covDiff is None)
820 def test_makeZeroSafe(self):
821 noZerosArray = [1.0, 20, -35, 45578.98, 90.0, 897, 659.8]
822 someZerosArray = [1.0, 20, 0, 0, 90, 879, 0]
823 allZerosArray = [0.0, 0.0, 0, 0, 0.0, 0, 0]
825 substituteValue = 1e-10
827 expectedSomeZerosArray = [
828 1.0,
829 20,
830 substituteValue,
831 substituteValue,
832 90,
833 879,
834 substituteValue,
835 ]
836 expectedAllZerosArray = np.repeat(substituteValue, len(allZerosArray))
838 measuredSomeZerosArray = self.defaultTaskSolve._makeZeroSafe(
839 someZerosArray, substituteValue=substituteValue
840 )
841 measuredAllZerosArray = self.defaultTaskSolve._makeZeroSafe(
842 allZerosArray, substituteValue=substituteValue
843 )
844 measuredNoZerosArray = self.defaultTaskSolve._makeZeroSafe(
845 noZerosArray, substituteValue=substituteValue
846 )
848 for exp, meas in zip(expectedSomeZerosArray, measuredSomeZerosArray):
849 self.assertEqual(exp, meas)
850 for exp, meas in zip(expectedAllZerosArray, measuredAllZerosArray):
851 self.assertEqual(exp, meas)
852 for exp, meas in zip(noZerosArray, measuredNoZerosArray):
853 self.assertEqual(exp, meas)
855 def test_getInitialGoodPoints(self):
856 xs = [1, 2, 3, 4, 5, 6]
857 ys = [2 * x for x in xs]
858 points = self.defaultTaskSolve._getInitialGoodPoints(
859 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2
860 )
861 assert np.all(points) == np.all(np.array([True for x in xs]))
863 ys[4] = 7 # Variance decreases in two consecutive points after ys[3]=8
864 ys[5] = 6
865 points = self.defaultTaskSolve._getInitialGoodPoints(
866 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2
867 )
868 assert np.all(points) == np.all(np.array([True, True, True, True, False]))
870 def runGetGainFromFlatPair(self, correctionType="NONE"):
871 extractConfig = self.defaultConfigExtract
872 extractConfig.gainCorrectionType = correctionType
873 extractConfig.minNumberGoodPixelsForCovariance = 5000
874 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig)
876 expDict = {}
877 expIds = []
878 idCounter = 0
879 inputGain = self.gain # 1.5 e/ADU
880 for expTime in self.timeVec:
881 # Approximation works better at low flux, e.g., < 10000 ADU
882 mockExp1, mockExp2 = makeMockFlats(
883 expTime,
884 gain=inputGain,
885 readNoiseElectrons=np.sqrt(self.noiseSq),
886 fluxElectrons=100,
887 expId1=idCounter,
888 expId2=idCounter + 1,
889 )
890 mockExpRef1 = PretendRef(mockExp1)
891 mockExpRef2 = PretendRef(mockExp2)
892 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1))
893 expIds.append(idCounter)
894 expIds.append(idCounter + 1)
895 idCounter += 2
897 resultsExtract = extractTask.run(
898 inputExp=expDict,
899 inputDims=expIds,
900 taskMetadata=[self.metadataContents for x in expIds],
901 )
902 for exposurePair in resultsExtract.outputCovariances:
903 for ampName in self.ampNames:
904 if exposurePair.gain[ampName] is np.nan:
905 continue
906 self.assertAlmostEqual(
907 exposurePair.gain[ampName], inputGain, delta=0.04
908 )
910 def test_getGainFromFlatPair(self):
911 for gainCorrectionType in [
912 "NONE",
913 "SIMPLE",
914 "FULL",
915 ]:
916 self.runGetGainFromFlatPair(gainCorrectionType)
918 def test_ptcFitBootstrap(self):
919 """Test the bootstrap fit option for the PTC"""
920 for (fitType, order) in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('EXPAPPROXIMATION', None)]:
921 self.ptcFitAndCheckPtc(fitType=fitType, order=order, doFitBootstrap=True)
924class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase):
925 def setUp(self):
926 self.ptcData = PhotonTransferCurveDataset(["C00", "C01"], " ")
927 self.ptcData.inputExpIdPairs = {
928 "C00": [(123, 234), (345, 456), (567, 678)],
929 "C01": [(123, 234), (345, 456), (567, 678)],
930 }
932 def test_generalBehaviour(self):
933 test = PhotonTransferCurveDataset(["C00", "C01"], " ")
934 test.inputExpIdPairs = {
935 "C00": [(123, 234), (345, 456), (567, 678)],
936 "C01": [(123, 234), (345, 456), (567, 678)],
937 }
940class TestMemory(lsst.utils.tests.MemoryTestCase):
941 pass
944def setup_module(module):
945 lsst.utils.tests.init()
948if __name__ == "__main__": 948 ↛ 949line 948 didn't jump to line 949, because the condition on line 948 was never true
949 lsst.utils.tests.init()
950 unittest.main()