Coverage for tests/test_ptc.py: 9%
356 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-04 15:28 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-04 15:28 -0700
1#!/usr/bin/env python
3#
4# LSST Data Management System
5#
6# Copyright 2008-2017 AURA/LSST.
7#
8# This product includes software developed by the
9# LSST Project (http://www.lsst.org/).
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the LSST License Statement and
22# the GNU General Public License along with this program. If not,
23# see <https://www.lsstcorp.org/LegalNotices/>.
24#
25"""Test cases for cp_pipe."""
27import unittest
28import numpy as np
29import copy
30import tempfile
31import logging
33import lsst.utils
34import lsst.utils.tests
36import lsst.cp.pipe as cpPipe
37import lsst.ip.isr.isrMock as isrMock
38from lsst.ip.isr import PhotonTransferCurveDataset
39from lsst.cp.pipe.utils import makeMockFlats
41from lsst.pipe.base import TaskMetadata
44class FakeCamera(list):
45 def getName(self):
46 return "FakeCam"
49class PretendRef:
50 "A class to act as a mock exposure reference"
52 def __init__(self, exposure):
53 self.exp = exposure
55 def get(self, component=None):
56 if component == "visitInfo":
57 return self.exp.getVisitInfo()
58 elif component == "detector":
59 return self.exp.getDetector()
60 elif component == "metadata":
61 return self.exp.getMetadata()
62 else:
63 return self.exp
66class MeasurePhotonTransferCurveTaskTestCase(lsst.utils.tests.TestCase):
67 """A test case for the PTC tasks."""
69 def setUp(self):
70 self.defaultConfigExtract = (
71 cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
72 )
73 self.defaultTaskExtract = cpPipe.ptc.PhotonTransferCurveExtractTask(
74 config=self.defaultConfigExtract
75 )
77 self.defaultConfigSolve = cpPipe.ptc.PhotonTransferCurveSolveTask.ConfigClass()
78 self.defaultTaskSolve = cpPipe.ptc.PhotonTransferCurveSolveTask(
79 config=self.defaultConfigSolve
80 )
82 self.flatMean = 2000
83 self.readNoiseAdu = 10
84 mockImageConfig = isrMock.IsrMock.ConfigClass()
86 # flatDrop is not really relevant as we replace the data
87 # but good to note it in case we change how this image is made
88 mockImageConfig.flatDrop = 0.99999
89 mockImageConfig.isTrimmed = True
91 self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
92 self.flatExp2 = self.flatExp1.clone()
93 (shapeY, shapeX) = self.flatExp1.getDimensions()
95 self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
97 self.rng1 = np.random.RandomState(1984)
98 flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
99 self.rng2 = np.random.RandomState(666)
100 flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
102 self.flatExp1.image.array[:] = flatData1
103 self.flatExp2.image.array[:] = flatData2
105 # create fake PTC data to see if fit works, for one amp ('amp')
106 self.flux = 1000.0 # ADU/sec
107 self.timeVec = np.arange(1.0, 101.0, 5)
108 self.k2NonLinearity = -5e-6
109 # quadratic signal-chain non-linearity
110 muVec = self.flux * self.timeVec + self.k2NonLinearity * self.timeVec**2
111 self.gain = 0.75 # e-/ADU
112 self.c1 = 1.0 / self.gain
113 self.noiseSq = 2 * self.gain # 7.5 (e-)^2
114 self.a00 = -1.2e-6
115 self.c2 = -1.5e-6
116 self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean
118 self.ampNames = [
119 amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()
120 ]
121 self.dataset = PhotonTransferCurveDataset(self.ampNames, ptcFitType="PARTIAL")
122 self.covariancesSqrtWeights = {}
123 for (
124 ampName
125 ) in self.ampNames: # just the expTimes and means here - vars vary per function
126 self.dataset.rawExpTimes[ampName] = self.timeVec
127 self.dataset.rawMeans[ampName] = muVec
128 self.dataset.covariancesSqrtWeights[ampName] = np.zeros(
129 (1, self.dataset.covMatrixSide, self.dataset.covMatrixSide)
130 )
132 # ISR metadata
133 self.metadataContents = TaskMetadata()
134 self.metadataContents["isr"] = {}
135 # Overscan readout noise [in ADU]
136 for amp in self.ampNames:
137 self.metadataContents["isr"][f"RESIDUAL STDEV {amp}"] = (
138 np.sqrt(self.noiseSq) / self.gain
139 )
141 def test_covAstier(self):
142 """Test to check getCovariancesAstier
144 We check that the gain is the same as the imput gain from the
145 mock data, that the covariances via FFT (as it is in
146 MeasurePhotonTransferCurveTask when doCovariancesAstier=True)
147 are the same as calculated in real space, and that Cov[0, 0]
148 (i.e., the variances) are similar to the variances calculated
149 with the standard method (when doCovariancesAstier=false),
151 """
152 extractConfig = self.defaultConfigExtract
153 extractConfig.minNumberGoodPixelsForCovariance = 5000
154 extractConfig.detectorMeasurementRegion = "FULL"
155 extractConfig.auxiliaryHeaderKeys = ["CCOBCURR", "CCDTEMP"]
156 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig)
158 solveConfig = self.defaultConfigSolve
159 solveConfig.ptcFitType = "FULLCOVARIANCE"
160 # Cut off the low-flux point which is a bad fit, and this
161 # also exercises this functionality and makes the tests
162 # run a lot faster.
163 solveConfig.minMeanSignal["ALL_AMPS"] = 2000.0
164 # Set the outlier fit threshold higher than the default appropriate
165 # for this test dataset.
166 solveConfig.maxSignalInitialPtcOutlierFit = 90000.0
167 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=solveConfig)
169 inputGain = self.gain
171 muStandard, varStandard = {}, {}
172 expDict = {}
173 expIds = []
174 idCounter = 0
175 for expTime in self.timeVec:
176 mockExp1, mockExp2 = makeMockFlats(
177 expTime,
178 gain=inputGain,
179 readNoiseElectrons=3,
180 expId1=idCounter,
181 expId2=idCounter + 1,
182 )
183 for mockExp in [mockExp1, mockExp2]:
184 md = mockExp.getMetadata()
185 # These values are chosen to be easily compared after
186 # processing for correct ordering.
187 md['CCOBCURR'] = float(idCounter)
188 md['CCDTEMP'] = float(idCounter + 1)
189 mockExp.setMetadata(md)
191 mockExpRef1 = PretendRef(mockExp1)
192 mockExpRef2 = PretendRef(mockExp2)
193 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1))
194 expIds.append(idCounter)
195 expIds.append(idCounter + 1)
196 for ampNumber, ampName in enumerate(self.ampNames):
197 # cov has (i, j, var, cov, npix)
198 (
199 im1Area,
200 im2Area,
201 imStatsCtrl,
202 mu1,
203 mu2,
204 ) = extractTask.getImageAreasMasksStats(mockExp1, mockExp2)
205 muDiff, varDiff, covAstier = extractTask.measureMeanVarCov(
206 im1Area, im2Area, imStatsCtrl, mu1, mu2
207 )
208 muStandard.setdefault(ampName, []).append(muDiff)
209 varStandard.setdefault(ampName, []).append(varDiff)
210 idCounter += 2
212 resultsExtract = extractTask.run(
213 inputExp=expDict,
214 inputDims=expIds,
215 taskMetadata=[self.metadataContents for x in expIds],
216 )
218 # Force the last PTC dataset to have a NaN, and ensure that the
219 # task runs (DM-38029). This is a minor perturbation and does not
220 # affect the output comparison. Note that we use index -2 because
221 # these datasets are in pairs of [real, dummy] to match the inputs
222 # to the extract task.
223 resultsExtract.outputCovariances[-2].rawMeans["C:0,0"] = np.array([np.nan])
224 resultsExtract.outputCovariances[-2].rawVars["C:0,0"] = np.array([np.nan])
226 # Force the next-to-last PTC dataset to have a decreased variance to
227 # ensure that the outlier fit rejection works. Note that we use
228 # index -4 because these datasets are in pairs of [real, dummy] to
229 # match the inputs to the extract task.
230 rawVar = resultsExtract.outputCovariances[-4].rawVars["C:0,0"]
231 resultsExtract.outputCovariances[-4].rawVars["C:0,0"] = rawVar * 0.9
233 # Reorganize the outputCovariances so we can confirm they come
234 # out sorted afterwards.
235 outputCovariancesRev = resultsExtract.outputCovariances[::-1]
237 resultsSolve = solveTask.run(
238 outputCovariancesRev, camera=FakeCamera([self.flatExp1.getDetector()])
239 )
241 ptc = resultsSolve.outputPtcDataset
243 # Some expected values for noise matrix, just to check that
244 # it was calculated.
245 noiseMatrixNoBExpected = {
246 (0, 0): 6.53126505,
247 (1, 1): -23.20924747,
248 (2, 2): 35.69834113,
249 }
250 noiseMatrixExpected = {
251 (0, 0): 29.37146918,
252 (1, 1): -14.6849025,
253 (2, 2): 24.7328517,
254 }
256 noiseMatrixExpected = np.array(
257 [
258 [
259 29.37146918,
260 9.2760363,
261 -29.08907932,
262 33.65818827,
263 -52.65710984,
264 -18.5821773,
265 -46.26896286,
266 65.01049736,
267 ],
268 [
269 -3.62427987,
270 -14.6849025,
271 -46.55230305,
272 -1.30410627,
273 6.44903599,
274 18.11796075,
275 -22.72874074,
276 20.90219857,
277 ],
278 [
279 5.09203058,
280 -4.40097862,
281 24.7328517,
282 39.2847586,
283 -21.46132351,
284 8.12179783,
285 6.23585617,
286 -2.09949622,
287 ],
288 [
289 35.79204016,
290 -6.50205005,
291 3.37910363,
292 15.22335662,
293 -19.29035067,
294 9.66065941,
295 7.47510934,
296 20.25962845,
297 ],
298 [
299 -36.23187633,
300 -22.72307472,
301 16.29140749,
302 -13.09493835,
303 3.32091085,
304 52.4380977,
305 -8.06428902,
306 -22.66669839,
307 ],
308 [
309 -27.93122896,
310 15.37016686,
311 9.18835073,
312 -24.48892946,
313 8.14480304,
314 22.38983222,
315 22.36866891,
316 -0.38803439,
317 ],
318 [
319 17.13962665,
320 -28.33153763,
321 -17.79744334,
322 -18.57064463,
323 7.69408833,
324 8.48265396,
325 18.0447022,
326 -16.97496022,
327 ],
328 [
329 10.09078383,
330 -26.61613002,
331 10.48504889,
332 15.33196998,
333 -23.35165517,
334 -24.53098643,
335 -18.21201067,
336 17.40755051,
337 ],
338 ]
339 )
341 noiseMatrixNoBExpected = np.array(
342 [
343 [
344 6.53126505,
345 12.14827594,
346 -37.11919923,
347 41.18675353,
348 -85.1613845,
349 -28.45801954,
350 -61.24442999,
351 88.76480122,
352 ],
353 [
354 -4.64541165,
355 -23.20924747,
356 -66.08733987,
357 -0.87558055,
358 12.20111853,
359 24.84795549,
360 -34.92458788,
361 24.42745014,
362 ],
363 [
364 7.66734507,
365 -4.51403645,
366 35.69834113,
367 52.73693356,
368 -30.85044089,
369 10.86761771,
370 10.8503068,
371 -2.18908327,
372 ],
373 [
374 50.9901156,
375 -7.34803977,
376 5.33443765,
377 21.60899396,
378 -25.06129827,
379 15.14015505,
380 10.94263771,
381 29.23975515,
382 ],
383 [
384 -48.66912069,
385 -31.58003774,
386 21.81305735,
387 -13.08993444,
388 8.17275394,
389 74.85293723,
390 -11.18403252,
391 -31.7799437,
392 ],
393 [
394 -38.55206382,
395 22.92982676,
396 13.39861008,
397 -33.3307362,
398 8.65362238,
399 29.18775548,
400 31.78433947,
401 1.27923706,
402 ],
403 [
404 23.33663918,
405 -41.74105625,
406 -26.55920751,
407 -24.71611677,
408 12.13343146,
409 11.25763907,
410 21.79131019,
411 -26.579393,
412 ],
413 [
414 11.44334226,
415 -34.9759641,
416 13.96449509,
417 19.64121933,
418 -36.09794843,
419 -34.27205933,
420 -25.16574105,
421 23.80460972,
422 ],
423 ]
424 )
426 for amp in self.ampNames:
427 self.assertAlmostEqual(ptc.gain[amp], inputGain, places=2)
428 for v1, v2 in zip(varStandard[amp], ptc.finalVars[amp]):
429 self.assertAlmostEqual(v1 / v2, 1.0, places=1)
431 # Check that the PTC turnoff is correctly computed.
432 # This will be different for the C:0,0 amp.
433 if amp == "C:0,0":
434 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-3])
435 else:
436 self.assertAlmostEqual(ptc.ptcTurnoff[amp], ptc.rawMeans[ampName][-1])
438 # Test that all the quantities are correctly ordered and have
439 # not accidentally been masked. We check every other output ([::2])
440 # because these datasets are in pairs of [real, dummy] to
441 # match the inputs to the extract task.
442 for i, extractPtc in enumerate(resultsExtract.outputCovariances[::2]):
443 self.assertFloatsAlmostEqual(
444 extractPtc.rawExpTimes[ampName][0],
445 ptc.rawExpTimes[ampName][i],
446 )
447 self.assertFloatsAlmostEqual(
448 extractPtc.rawMeans[ampName][0],
449 ptc.rawMeans[ampName][i],
450 )
451 self.assertFloatsAlmostEqual(
452 extractPtc.rawVars[ampName][0],
453 ptc.rawVars[ampName][i],
454 )
455 self.assertFloatsAlmostEqual(
456 extractPtc.histVars[ampName][0],
457 ptc.histVars[ampName][i],
458 )
459 self.assertFloatsAlmostEqual(
460 extractPtc.histChi2Dofs[ampName][0],
461 ptc.histChi2Dofs[ampName][i],
462 )
463 self.assertFloatsAlmostEqual(
464 extractPtc.kspValues[ampName][0],
465 ptc.kspValues[ampName][i],
466 )
467 self.assertFloatsAlmostEqual(
468 extractPtc.covariances[ampName][0],
469 ptc.covariances[ampName][i],
470 )
471 self.assertFloatsAlmostEqual(
472 extractPtc.covariancesSqrtWeights[ampName][0],
473 ptc.covariancesSqrtWeights[ampName][i],
474 )
475 self.assertFloatsAlmostEqual(
476 ptc.noiseMatrix[ampName], noiseMatrixExpected, atol=1e-8, rtol=None
477 )
478 self.assertFloatsAlmostEqual(
479 ptc.noiseMatrixNoB[ampName],
480 noiseMatrixNoBExpected,
481 atol=1e-8,
482 rtol=None,
483 )
485 mask = ptc.getGoodPoints(amp)
487 values = (
488 ptc.covariancesModel[amp][mask, 0, 0] - ptc.covariances[amp][mask, 0, 0]
489 ) / ptc.covariancesModel[amp][mask, 0, 0]
490 np.testing.assert_array_less(np.abs(values), 2e-3)
492 values = (
493 ptc.covariancesModel[amp][mask, 1, 1] - ptc.covariances[amp][mask, 1, 1]
494 ) / ptc.covariancesModel[amp][mask, 1, 1]
495 np.testing.assert_array_less(np.abs(values), 0.2)
497 values = (
498 ptc.covariancesModel[amp][mask, 1, 2] - ptc.covariances[amp][mask, 1, 2]
499 ) / ptc.covariancesModel[amp][mask, 1, 2]
500 np.testing.assert_array_less(np.abs(values), 0.2)
502 # And test that the auxiliary values are there and correctly ordered.
503 self.assertIn('CCOBCURR', ptc.auxValues)
504 self.assertIn('CCDTEMP', ptc.auxValues)
505 firstExpIds = np.array([i for i, _ in ptc.inputExpIdPairs['C:0,0']], dtype=np.float64)
506 self.assertFloatsAlmostEqual(ptc.auxValues['CCOBCURR'], firstExpIds)
507 self.assertFloatsAlmostEqual(ptc.auxValues['CCDTEMP'], firstExpIds + 1)
509 expIdsUsed = ptc.getExpIdsUsed("C:0,0")
510 # Check that these are the same as the inputs, paired up, with the
511 # first two (low flux) and final four (outliers, nans) removed.
512 self.assertTrue(
513 np.all(expIdsUsed == np.array(expIds).reshape(len(expIds) // 2, 2)[1:-2])
514 )
516 goodAmps = ptc.getGoodAmps()
517 self.assertEqual(goodAmps, self.ampNames)
519 # Check that every possibly modified field has the same length.
520 covShape = None
521 covSqrtShape = None
522 covModelShape = None
523 covModelNoBShape = None
525 for ampName in self.ampNames:
526 if covShape is None:
527 covShape = ptc.covariances[ampName].shape
528 covSqrtShape = ptc.covariancesSqrtWeights[ampName].shape
529 covModelShape = ptc.covariancesModel[ampName].shape
530 covModelNoBShape = ptc.covariancesModelNoB[ampName].shape
531 else:
532 self.assertEqual(ptc.covariances[ampName].shape, covShape)
533 self.assertEqual(
534 ptc.covariancesSqrtWeights[ampName].shape, covSqrtShape
535 )
536 self.assertEqual(ptc.covariancesModel[ampName].shape, covModelShape)
537 self.assertEqual(
538 ptc.covariancesModelNoB[ampName].shape, covModelNoBShape
539 )
541 # And check that this is serializable
542 with tempfile.NamedTemporaryFile(suffix=".fits") as f:
543 usedFilename = ptc.writeFits(f.name)
544 fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
545 self.assertEqual(fromFits, ptc)
547 def ptcFitAndCheckPtc(
548 self,
549 order=None,
550 fitType=None,
551 doFitBootstrap=False,
552 doLegacy=False,
553 ):
554 localDataset = copy.deepcopy(self.dataset)
555 localDataset.ptcFitType = fitType
556 configSolve = copy.copy(self.defaultConfigSolve)
557 if doFitBootstrap:
558 configSolve.doFitBootstrap = True
560 configSolve.doLegacyTurnoffSelection = doLegacy
562 if fitType == "POLYNOMIAL":
563 if order not in [2, 3]:
564 RuntimeError("Enter a valid polynomial order for this test: 2 or 3")
565 if order == 2:
566 for ampName in self.ampNames:
567 localDataset.rawVars[ampName] = [
568 self.noiseSq + self.c1 * mu + self.c2 * mu**2
569 for mu in localDataset.rawMeans[ampName]
570 ]
571 configSolve.polynomialFitDegree = 2
572 if order == 3:
573 for ampName in self.ampNames:
574 localDataset.rawVars[ampName] = [
575 self.noiseSq
576 + self.c1 * mu
577 + self.c2 * mu**2
578 + self.c3 * mu**3
579 for mu in localDataset.rawMeans[ampName]
580 ]
581 configSolve.polynomialFitDegree = 3
582 elif fitType == "EXPAPPROXIMATION":
583 g = self.gain
584 for ampName in self.ampNames:
585 localDataset.rawVars[ampName] = [
586 (
587 0.5 / (self.a00 * g**2) * (np.exp(2 * self.a00 * mu * g) - 1)
588 + self.noiseSq / (g * g)
589 )
590 for mu in localDataset.rawMeans[ampName]
591 ]
592 else:
593 raise RuntimeError(
594 "Enter a fit function type: 'POLYNOMIAL' or 'EXPAPPROXIMATION'"
595 )
597 # Initialize mask and covariance weights that will be used in fits.
598 # Covariance weights values empirically determined from one of
599 # the cases in test_covAstier.
600 matrixSize = localDataset.covMatrixSide
601 maskLength = len(localDataset.rawMeans[ampName])
602 for ampName in self.ampNames:
603 localDataset.expIdMask[ampName] = np.repeat(True, maskLength)
604 localDataset.covariancesSqrtWeights[ampName] = np.repeat(
605 np.ones((matrixSize, matrixSize)), maskLength
606 ).reshape((maskLength, matrixSize, matrixSize))
607 localDataset.covariancesSqrtWeights[ampName][:, 0, 0] = [
608 0.07980188,
609 0.01339653,
610 0.0073118,
611 0.00502802,
612 0.00383132,
613 0.00309475,
614 0.00259572,
615 0.00223528,
616 0.00196273,
617 0.00174943,
618 0.00157794,
619 0.00143707,
620 0.00131929,
621 0.00121935,
622 0.0011334,
623 0.00105893,
624 0.00099357,
625 0.0009358,
626 0.00088439,
627 0.00083833,
628 ]
630 solveTask = cpPipe.ptc.PhotonTransferCurveSolveTask(config=configSolve)
632 localDataset = solveTask.fitMeasurementsToModel(localDataset)
634 # check entries in localDataset, which was modified by the function
635 for ampName in self.ampNames:
636 self.assertEqual(fitType, localDataset.ptcFitType)
637 self.assertAlmostEqual(self.gain, localDataset.gain[ampName])
638 if fitType == "POLYNOMIAL":
639 self.assertAlmostEqual(self.c1, localDataset.ptcFitPars[ampName][1])
640 self.assertAlmostEqual(
641 np.sqrt(self.noiseSq) * self.gain, localDataset.noise[ampName]
642 )
643 if fitType == "EXPAPPROXIMATION":
644 self.assertAlmostEqual(
645 self.a00, localDataset.ptcFitPars[ampName][0]
646 )
647 # noise already in electrons for 'EXPAPPROXIMATION' fit
648 self.assertAlmostEqual(
649 np.sqrt(self.noiseSq), localDataset.noise[ampName]
650 )
652 def test_ptcFit(self):
653 for doLegacy in [False, True]:
654 for fitType, order in [
655 ("POLYNOMIAL", 2),
656 ("POLYNOMIAL", 3),
657 ("EXPAPPROXIMATION", None),
658 ]:
659 self.ptcFitAndCheckPtc(
660 fitType=fitType,
661 order=order,
662 doLegacy=doLegacy,
663 )
665 def test_meanVarMeasurement(self):
666 task = self.defaultTaskExtract
667 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
668 self.flatExp1, self.flatExp2
669 )
670 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
672 self.assertLess(self.flatWidth - np.sqrt(varDiff), 1)
673 self.assertLess(self.flatMean - mu, 1)
675 def test_meanVarMeasurementWithNans(self):
676 task = self.defaultTaskExtract
678 flatExp1 = self.flatExp1.clone()
679 flatExp2 = self.flatExp2.clone()
681 flatExp1.image.array[20:30, :] = np.nan
682 flatExp2.image.array[20:30, :] = np.nan
684 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
685 flatExp1, flatExp2
686 )
687 mu, varDiff, _ = task.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
689 expectedMu1 = np.nanmean(flatExp1.image.array)
690 expectedMu2 = np.nanmean(flatExp2.image.array)
691 expectedMu = 0.5 * (expectedMu1 + expectedMu2)
693 # Now the variance of the difference. First, create the diff image.
694 im1 = flatExp1.maskedImage
695 im2 = flatExp2.maskedImage
697 temp = im2.clone()
698 temp *= expectedMu1
699 diffIm = im1.clone()
700 diffIm *= expectedMu2
701 diffIm -= temp
702 diffIm /= expectedMu
704 # Divide by two as it is what measureMeanVarCov returns
705 # (variance of difference)
706 expectedVar = 0.5 * np.nanvar(diffIm.image.array)
708 # Check that the standard deviations and the emans agree to
709 # less than 1 ADU
710 self.assertLess(np.sqrt(expectedVar) - np.sqrt(varDiff), 1)
711 self.assertLess(expectedMu - mu, 1)
713 def test_meanVarMeasurementAllNan(self):
714 task = self.defaultTaskExtract
715 flatExp1 = self.flatExp1.clone()
716 flatExp2 = self.flatExp2.clone()
718 flatExp1.image.array[:, :] = np.nan
719 flatExp2.image.array[:, :] = np.nan
721 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
722 flatExp1, flatExp2
723 )
724 mu, varDiff, covDiff = task.measureMeanVarCov(
725 im1Area, im2Area, imStatsCtrl, mu1, mu2
726 )
728 self.assertTrue(np.isnan(mu))
729 self.assertTrue(np.isnan(varDiff))
730 self.assertTrue(covDiff is None)
732 def test_meanVarMeasurementTooFewPixels(self):
733 task = self.defaultTaskExtract
734 flatExp1 = self.flatExp1.clone()
735 flatExp2 = self.flatExp2.clone()
737 flatExp1.image.array[0:190, :] = np.nan
738 flatExp2.image.array[0:190, :] = np.nan
740 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"]
741 flatExp1.mask.array[0:190, :] &= 2**bit
742 flatExp2.mask.array[0:190, :] &= 2**bit
744 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
745 flatExp1, flatExp2
746 )
747 with self.assertLogs(level=logging.WARNING) as cm:
748 mu, varDiff, covDiff = task.measureMeanVarCov(
749 im1Area, im2Area, imStatsCtrl, mu1, mu2
750 )
751 self.assertIn("Number of good points", cm.output[0])
753 self.assertTrue(np.isnan(mu))
754 self.assertTrue(np.isnan(varDiff))
755 self.assertTrue(covDiff is None)
757 def test_meanVarMeasurementTooNarrowStrip(self):
758 # We need a new config to make sure the second covariance cut is
759 # triggered.
760 config = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
761 config.minNumberGoodPixelsForCovariance = 10
762 task = cpPipe.ptc.PhotonTransferCurveExtractTask(config=config)
763 flatExp1 = self.flatExp1.clone()
764 flatExp2 = self.flatExp2.clone()
766 flatExp1.image.array[0:195, :] = np.nan
767 flatExp2.image.array[0:195, :] = np.nan
768 flatExp1.image.array[:, 0:195] = np.nan
769 flatExp2.image.array[:, 0:195] = np.nan
771 bit = flatExp1.mask.getMaskPlaneDict()["NO_DATA"]
772 flatExp1.mask.array[0:195, :] &= 2**bit
773 flatExp2.mask.array[0:195, :] &= 2**bit
774 flatExp1.mask.array[:, 0:195] &= 2**bit
775 flatExp2.mask.array[:, 0:195] &= 2**bit
777 im1Area, im2Area, imStatsCtrl, mu1, mu2 = task.getImageAreasMasksStats(
778 flatExp1, flatExp2
779 )
780 with self.assertLogs(level=logging.WARNING) as cm:
781 mu, varDiff, covDiff = task.measureMeanVarCov(
782 im1Area, im2Area, imStatsCtrl, mu1, mu2
783 )
784 self.assertIn("Not enough pixels", cm.output[0])
786 self.assertTrue(np.isnan(mu))
787 self.assertTrue(np.isnan(varDiff))
788 self.assertTrue(covDiff is None)
790 def test_makeZeroSafe(self):
791 noZerosArray = [1.0, 20, -35, 45578.98, 90.0, 897, 659.8]
792 someZerosArray = [1.0, 20, 0, 0, 90, 879, 0]
793 allZerosArray = [0.0, 0.0, 0, 0, 0.0, 0, 0]
795 substituteValue = 1e-10
797 expectedSomeZerosArray = [
798 1.0,
799 20,
800 substituteValue,
801 substituteValue,
802 90,
803 879,
804 substituteValue,
805 ]
806 expectedAllZerosArray = np.repeat(substituteValue, len(allZerosArray))
808 measuredSomeZerosArray = self.defaultTaskSolve._makeZeroSafe(
809 someZerosArray, substituteValue=substituteValue
810 )
811 measuredAllZerosArray = self.defaultTaskSolve._makeZeroSafe(
812 allZerosArray, substituteValue=substituteValue
813 )
814 measuredNoZerosArray = self.defaultTaskSolve._makeZeroSafe(
815 noZerosArray, substituteValue=substituteValue
816 )
818 for exp, meas in zip(expectedSomeZerosArray, measuredSomeZerosArray):
819 self.assertEqual(exp, meas)
820 for exp, meas in zip(expectedAllZerosArray, measuredAllZerosArray):
821 self.assertEqual(exp, meas)
822 for exp, meas in zip(noZerosArray, measuredNoZerosArray):
823 self.assertEqual(exp, meas)
825 def test_getInitialGoodPoints(self):
826 xs = [1, 2, 3, 4, 5, 6]
827 ys = [2 * x for x in xs]
828 points = self.defaultTaskSolve._getInitialGoodPoints(
829 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2
830 )
831 assert np.all(points) == np.all(np.array([True for x in xs]))
833 ys[4] = 7 # Variance decreases in two consecutive points after ys[3]=8
834 ys[5] = 6
835 points = self.defaultTaskSolve._getInitialGoodPoints(
836 xs, ys, minVarPivotSearch=0.0, consecutivePointsVarDecreases=2
837 )
838 assert np.all(points) == np.all(np.array([True, True, True, True, False]))
840 def runGetGainFromFlatPair(self, correctionType="NONE"):
841 extractConfig = self.defaultConfigExtract
842 extractConfig.gainCorrectionType = correctionType
843 extractConfig.minNumberGoodPixelsForCovariance = 5000
844 extractTask = cpPipe.ptc.PhotonTransferCurveExtractTask(config=extractConfig)
846 expDict = {}
847 expIds = []
848 idCounter = 0
849 inputGain = self.gain # 1.5 e/ADU
850 for expTime in self.timeVec:
851 # Approximation works better at low flux, e.g., < 10000 ADU
852 mockExp1, mockExp2 = makeMockFlats(
853 expTime,
854 gain=inputGain,
855 readNoiseElectrons=np.sqrt(self.noiseSq),
856 fluxElectrons=100,
857 expId1=idCounter,
858 expId2=idCounter + 1,
859 )
860 mockExpRef1 = PretendRef(mockExp1)
861 mockExpRef2 = PretendRef(mockExp2)
862 expDict[expTime] = ((mockExpRef1, idCounter), (mockExpRef2, idCounter + 1))
863 expIds.append(idCounter)
864 expIds.append(idCounter + 1)
865 idCounter += 2
867 resultsExtract = extractTask.run(
868 inputExp=expDict,
869 inputDims=expIds,
870 taskMetadata=[self.metadataContents for x in expIds],
871 )
872 for exposurePair in resultsExtract.outputCovariances:
873 for ampName in self.ampNames:
874 if exposurePair.gain[ampName] is np.nan:
875 continue
876 self.assertAlmostEqual(
877 exposurePair.gain[ampName], inputGain, delta=0.04
878 )
880 def test_getGainFromFlatPair(self):
881 for gainCorrectionType in [
882 "NONE",
883 "SIMPLE",
884 "FULL",
885 ]:
886 self.runGetGainFromFlatPair(gainCorrectionType)
888 def test_ptcFitBootstrap(self):
889 """Test the bootstrap fit option for the PTC"""
890 for (fitType, order) in [('POLYNOMIAL', 2), ('POLYNOMIAL', 3), ('EXPAPPROXIMATION', None)]:
891 self.ptcFitAndCheckPtc(fitType=fitType, order=order, doFitBootstrap=True)
894class MeasurePhotonTransferCurveDatasetTestCase(lsst.utils.tests.TestCase):
895 def setUp(self):
896 self.ptcData = PhotonTransferCurveDataset(["C00", "C01"], " ")
897 self.ptcData.inputExpIdPairs = {
898 "C00": [(123, 234), (345, 456), (567, 678)],
899 "C01": [(123, 234), (345, 456), (567, 678)],
900 }
902 def test_generalBehaviour(self):
903 test = PhotonTransferCurveDataset(["C00", "C01"], " ")
904 test.inputExpIdPairs = {
905 "C00": [(123, 234), (345, 456), (567, 678)],
906 "C01": [(123, 234), (345, 456), (567, 678)],
907 }
910class TestMemory(lsst.utils.tests.MemoryTestCase):
911 pass
914def setup_module(module):
915 lsst.utils.tests.init()
918if __name__ == "__main__": 918 ↛ 919line 918 didn't jump to line 919, because the condition on line 918 was never true
919 lsst.utils.tests.init()
920 unittest.main()