23 import matplotlib.pyplot
as plt
24 from collections
import Counter
29 from .utils
import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30 from scipy.optimize
import least_squares
34 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect, fitData)
35 from .linearity
import LinearitySolveTask
36 from .photodiode
import getBOTphotodiodeData
38 from lsst.pipe.tasks.getRepositoryData
import DataRefListRunner
41 __all__ = [
'MeasurePhotonTransferCurveTask',
42 'MeasurePhotonTransferCurveTaskConfig']
46 """Config class for photon transfer curve measurement task"""
47 ccdKey = pexConfig.Field(
49 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
52 ptcFitType = pexConfig.ChoiceField(
54 doc=
"Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
57 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
58 "EXPAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16).",
59 "FULLCOVARIANCE":
"Full covariances model in Astier+19 (Eq. 20)"
62 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
64 doc=
"sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
67 maxIterFullFitCovariancesAstier = pexConfig.Field(
69 doc=
"Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
72 maximumRangeCovariancesAstier = pexConfig.Field(
74 doc=
"Maximum range of covariances as in Astier+19",
77 covAstierRealSpace = pexConfig.Field(
79 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
82 polynomialFitDegree = pexConfig.Field(
84 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
87 linearity = pexConfig.ConfigurableField(
88 target=LinearitySolveTask,
89 doc=
"Task to solve the linearity."
92 doCreateLinearizer = pexConfig.Field(
94 doc=
"Calculate non-linearity and persist linearizer?",
98 binSize = pexConfig.Field(
100 doc=
"Bin the image by this factor in both dimensions.",
103 minMeanSignal = pexConfig.Field(
105 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
108 maxMeanSignal = pexConfig.Field(
110 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
113 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
115 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
116 " linear in the positive direction, from the PTC fit. Note that these points will also be"
117 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
118 " to allow an accurate determination of the sigmas for said iterative fit.",
123 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
125 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
126 " linear in the negative direction, from the PTC fit. Note that these points will also be"
127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128 " to allow an accurate determination of the sigmas for said iterative fit.",
133 sigmaCutPtcOutliers = pexConfig.Field(
135 doc=
"Sigma cut for outlier rejection in PTC.",
138 maskNameList = pexConfig.ListField(
140 doc=
"Mask list to exclude from statistics calculations.",
141 default=[
'SUSPECT',
'BAD',
'NO_DATA'],
143 nSigmaClipPtc = pexConfig.Field(
145 doc=
"Sigma cut for afwMath.StatisticsControl()",
148 nIterSigmaClipPtc = pexConfig.Field(
150 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
153 maxIterationsPtcOutliers = pexConfig.Field(
155 doc=
"Maximum number of iterations for outlier rejection in PTC.",
158 doFitBootstrap = pexConfig.Field(
160 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
163 doPhotodiode = pexConfig.Field(
165 doc=
"Apply a correction based on the photodiode readings if available?",
168 photodiodeDataPath = pexConfig.Field(
170 doc=
"Gen2 only: path to locate the data photodiode data files.",
173 instrumentName = pexConfig.Field(
175 doc=
"Instrument name.",
181 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
183 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
184 used in astronomical detectors characterization (e.g., Janesick 2001,
185 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
186 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
187 times. The difference image of each pair is formed to eliminate fixed pattern noise,
188 and then the variance of the difference image and the mean of the average image
189 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
190 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
191 arXiv:1905.08677) can be fitted to the PTC curve. These models include
192 parameters such as the gain (e/DN) and readout noise.
194 Linearizers to correct for signal-chain non-linearity are also calculated.
195 The `Linearizer` class, in general, can support per-amp linearizers, but in this
196 task this is not supported.
198 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
199 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
200 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
207 Positional arguments passed to the Task constructor. None used at this
210 Keyword arguments passed on to the Task constructor. None used at this
215 RunnerClass = DataRefListRunner
216 ConfigClass = MeasurePhotonTransferCurveTaskConfig
217 _DefaultName =
"measurePhotonTransferCurve"
220 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
221 self.makeSubtask(
"linearity")
222 plt.interactive(
False)
223 self.config.validate()
228 """Run the Photon Transfer Curve (PTC) measurement task.
230 For a dataRef (which is each detector here),
231 and given a list of exposure pairs (postISR) at different exposure times,
236 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
237 Data references for exposures for detectors to process.
239 if len(dataRefList) < 2:
240 raise RuntimeError(
"Insufficient inputs to combine.")
243 dataRef = dataRefList[0]
245 detNum = dataRef.dataId[self.config.ccdKey]
246 camera = dataRef.get(
'camera')
247 detector = camera[dataRef.dataId[self.config.ccdKey]]
249 amps = detector.getAmplifiers()
250 ampNames = [amp.getName()
for amp
in amps]
251 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
256 for (exp1, exp2)
in expPairs.values():
257 id1 = exp1.getInfo().getVisitInfo().getExposureId()
258 id2 = exp2.getInfo().getVisitInfo().getExposureId()
259 expIds.append((id1, id2))
260 self.log.info(f
"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
265 if self.config.doPhotodiode:
266 for (expId1, expId2)
in expIds:
268 for i, expId
in enumerate([expId1, expId2]):
273 dataRef.dataId[
'expId'] = expId//1000
274 if self.config.photodiodeDataPath:
279 charges[i] = photodiodeData.getCharge()
283 self.log.warn(f
"No photodiode data found for {expId}")
285 for ampName
in ampNames:
286 datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
290 for ampName
in ampNames:
291 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
293 for ampName
in ampNames:
294 datasetPtc.inputExpIdPairs[ampName] = expIds
298 for expTime, (exp1, exp2)
in expPairs.items():
299 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
300 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
303 for ampNumber, amp
in enumerate(detector):
304 ampName = amp.getName()
306 doRealSpace = self.config.covAstierRealSpace
307 muDiff, varDiff, covAstier = self.
measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
308 covAstierRealSpace=doRealSpace)
309 datasetPtc.rawExpTimes[ampName].append(expTime)
310 datasetPtc.rawMeans[ampName].append(muDiff)
311 datasetPtc.rawVars[ampName].append(varDiff)
313 if np.isnan(muDiff)
or np.isnan(varDiff)
or (covAstier
is None):
314 msg = (f
"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
315 f
" {expId2} of detector {detNum}.")
319 tags = [
'mu',
'i',
'j',
'var',
'cov',
'npix',
'ext',
'expTime',
'ampName']
320 if (muDiff <= self.config.minMeanSignal)
or (muDiff >= self.config.maxMeanSignal):
323 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName)
for covRow
in covAstier]
324 if nAmpsNan == len(ampNames):
325 msg = f
"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
329 tupleRecords += tupleRows
330 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
332 if self.config.ptcFitType
in [
"FULLCOVARIANCE", ]:
335 elif self.config.ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
338 datasetPtc = self.
fitPtc(datasetPtc, self.config.ptcFitType)
340 detName = detector.getName()
341 now = datetime.datetime.utcnow()
342 calibDate = now.strftime(
"%Y-%m-%d")
343 butler = dataRef.getButler()
345 datasetPtc.updateMetadata(setDate=
True, camera=camera, detector=detector)
348 if self.config.doCreateLinearizer:
354 dimensions = {
'camera': camera.getName(),
'detector': detector.getId()}
355 linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
356 linearizer = linearityResults.outputLinearizer
358 self.log.info(
"Writing linearizer:")
359 butler.put(linearizer, datasetType=
'Linearizer', dataId={
'detector': detNum,
360 'detectorName': detName,
'calibDate': calibDate})
362 self.log.info(f
"Writing PTC data.")
363 butler.put(datasetPtc, datasetType=
'photonTransferCurveDataset', dataId={
'detector': detNum,
364 'detectorName': detName,
'calibDate': calibDate})
366 return pipeBase.Struct(exitStatus=0)
369 """Produce a list of flat pairs indexed by exposure time.
373 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
374 Data references for exposures for detectors to process.
378 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
379 Dictionary that groups flat-field exposures that have the same exposure time (seconds).
383 We use the difference of one pair of flat-field images taken at the same exposure time when
384 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
385 same exposure time, the first two are kept and the rest discarded.
390 for dataRef
in dataRefList:
392 tempFlat = dataRef.get(
"postISRCCD")
394 self.log.warn(
"postISR exposure could not be retrieved. Ignoring flat.")
396 expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
397 expDict.setdefault(expDate, tempFlat)
398 sortedExps = {k: expDict[k]
for k
in sorted(expDict)}
401 for exp
in sortedExps:
402 tempFlat = sortedExps[exp]
403 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
404 listAtExpTime = flatPairs.setdefault(expTime, [])
405 if len(listAtExpTime) < 2:
406 listAtExpTime.append(tempFlat)
407 if len(listAtExpTime) > 2:
408 self.log.warn(f
"More than 2 exposures found at expTime {expTime}. Dropping exposures "
409 f
"{listAtExpTime[2:]}.")
411 for (key, value)
in flatPairs.items():
414 self.log.warn(f
"Only one exposure found at expTime {key}. Dropping exposure {value}.")
418 """Fit measured flat covariances to full model in Astier+19.
422 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
423 The dataset containing information such as the means, variances and exposure times.
425 covariancesWithTagsArray : `numpy.recarray`
426 Tuple with at least (mu, cov, var, i, j, npix), where:
427 mu : 0.5*(m1 + m2), where:
428 mu1: mean value of flat1
429 mu2: mean value of flat2
430 cov: covariance value at lag(i, j)
431 var: variance(covariance value at lag(0, 0))
434 npix: number of pixels used for covariance calculation.
438 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
439 This is the same dataset as the input paramter, however, it has been modified
440 to include information such as the fit vectors and the fit parameters. See
441 the class `PhotonTransferCurveDatase`.
444 covFits, covFitsNoB =
fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
445 r=self.config.maximumRangeCovariancesAstier,
446 nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
447 maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
454 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
458 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
459 The dataset containing information such as the means, variances and exposure times.
462 Dictionary of CovFit objects, with amp names as keys.
465 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
469 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
470 This is the same dataset as the input paramter, however, it has been modified
471 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
472 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
473 See the class `PhotonTransferCurveDatase`.
475 assert(len(covFits) == len(covFitsNoB))
477 for i, amp
in enumerate(dataset.ampNames):
478 lenInputTimes = len(dataset.rawExpTimes[amp])
480 dataset.ptcFitPars[amp] = np.nan
481 dataset.ptcFitParsError[amp] = np.nan
482 dataset.ptcFitChiSq[amp] = np.nan
485 fitNoB = covFitsNoB[amp]
487 dataset.covariances[amp] = fit.cov
488 dataset.covariancesModel[amp] = fit.evalCovModel()
489 dataset.covariancesSqrtWeights[amp] = fit.sqrtW
490 dataset.aMatrix[amp] = fit.getA()
491 dataset.bMatrix[amp] = fit.getB()
492 dataset.covariancesNoB[amp] = fitNoB.cov
493 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
494 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
495 dataset.aMatrixNoB[amp] = fitNoB.getA()
497 (meanVecFinal, varVecFinal, varVecModel,
498 wc, varMask) = fit.getFitData(0, 0, divideByMu=
False, returnMasked=
True)
500 dataset.expIdMask[amp] = varMask
501 dataset.gain[amp] = gain
502 dataset.gainErr[amp] = fit.getGainErr()
503 dataset.noise[amp] = np.sqrt(fit.getRon())
504 dataset.noiseErr[amp] = fit.getRonErr()
506 padLength = lenInputTimes - len(varVecFinal)
507 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength),
'constant',
508 constant_values=np.nan)
509 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength),
'constant',
510 constant_values=np.nan)
511 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength),
'constant',
512 constant_values=np.nan)
516 matrixSide = self.config.maximumRangeCovariancesAstier
517 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
518 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
520 dataset.covariances[amp] = listNanMatrix
521 dataset.covariancesModel[amp] = listNanMatrix
522 dataset.covariancesSqrtWeights[amp] = listNanMatrix
523 dataset.aMatrix[amp] = nanMatrix
524 dataset.bMatrix[amp] = nanMatrix
525 dataset.covariancesNoB[amp] = listNanMatrix
526 dataset.covariancesModelNoB[amp] = listNanMatrix
527 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
528 dataset.aMatrixNoB[amp] = nanMatrix
530 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
531 dataset.gain[amp] = np.nan
532 dataset.gainErr[amp] = np.nan
533 dataset.noise[amp] = np.nan
534 dataset.noiseErr[amp] = np.nan
535 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
536 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
537 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
542 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
544 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
545 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
546 keep one (covariance).
550 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
551 First exposure of flat field pair.
553 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
554 Second exposure of flat field pair.
556 region : `lsst.geom.Box2I`, optional
557 Region of each exposure where to perform the calculations (e.g, an amplifier).
559 covAstierRealSpace : `bool`, optional
560 Should the covariannces in Astier+19 be calculated in real space or via FFT?
561 See Appendix A of Astier+19.
565 mu : `float` or `NaN`
566 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
567 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
569 varDiff : `float` or `NaN`
570 Half of the clipped variance of the difference of the regions inthe two input
571 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
573 covDiffAstier : `list` or `NaN`
574 List with tuples of the form (dx, dy, var, cov, npix), where:
580 Variance at (dx, dy).
582 Covariance at (dx, dy).
584 Number of pixel pairs used to evaluate var and cov.
585 If either mu1 or m2 are NaN's, the returned value is NaN.
588 if region
is not None:
589 im1Area = exposure1.maskedImage[region]
590 im2Area = exposure2.maskedImage[region]
592 im1Area = exposure1.maskedImage
593 im2Area = exposure2.maskedImage
595 if self.config.binSize > 1:
596 im1Area = afwMath.binImage(im1Area, self.config.binSize)
597 im2Area = afwMath.binImage(im2Area, self.config.binSize)
599 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
600 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
601 self.config.nIterSigmaClipPtc,
603 im1StatsCtrl.setNanSafe(
True)
604 im1StatsCtrl.setAndMask(im1MaskVal)
606 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
607 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
608 self.config.nIterSigmaClipPtc,
610 im2StatsCtrl.setNanSafe(
True)
611 im2StatsCtrl.setAndMask(im2MaskVal)
614 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
615 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
616 if np.isnan(mu1)
or np.isnan(mu2):
617 return np.nan, np.nan,
None
622 temp = im2Area.clone()
624 diffIm = im1Area.clone()
629 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
630 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
631 self.config.nIterSigmaClipPtc,
633 diffImStatsCtrl.setNanSafe(
True)
634 diffImStatsCtrl.setAndMask(diffImMaskVal)
636 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
639 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
640 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
643 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
646 maxRangeCov = self.config.maximumRangeCovariancesAstier
647 if covAstierRealSpace:
648 covDiffAstier =
computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
650 shapeDiff = diffIm.getImage().getArray().shape
651 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
652 c =
CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
653 covDiffAstier = c.reportCovFft(maxRangeCov)
655 return mu, varDiff, covDiffAstier
658 """Compute covariances of diffImage in real space.
660 For lags larger than ~25, it is slower than the FFT way.
661 Taken from https://github.com/PierreAstier/bfptc/
665 diffImage : `numpy.array`
666 Image to compute the covariance of.
668 weightImage : `numpy.array`
669 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
672 Last index of the covariance to be computed.
677 List with tuples of the form (dx, dy, var, cov, npix), where:
683 Variance at (dx, dy).
685 Covariance at (dx, dy).
687 Number of pixel pairs used to evaluate var and cov.
692 for dy
in range(maxRange + 1):
693 for dx
in range(0, maxRange + 1):
696 cov2, nPix2 = self.
covDirectValue(diffImage, weightImage, dx, -dy)
697 cov = 0.5*(cov1 + cov2)
701 if (dx == 0
and dy == 0):
703 outList.append((dx, dy, var, cov, nPix))
708 """Compute covariances of diffImage in real space at lag (dx, dy).
710 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
714 diffImage : `numpy.array`
715 Image to compute the covariance of.
717 weightImage : `numpy.array`
718 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
729 Covariance at (dx, dy)
732 Number of pixel pairs used to evaluate var and cov.
734 (nCols, nRows) = diffImage.shape
738 (dx, dy) = (-dx, -dy)
742 im1 = diffImage[dy:, dx:]
743 w1 = weightImage[dy:, dx:]
744 im2 = diffImage[:nCols - dy, :nRows - dx]
745 w2 = weightImage[:nCols - dy, :nRows - dx]
747 im1 = diffImage[:nCols + dy, dx:]
748 w1 = weightImage[:nCols + dy, dx:]
749 im2 = diffImage[-dy:, :nRows - dx]
750 w2 = weightImage[-dy:, :nRows - dx]
756 s1 = im1TimesW.sum()/nPix
757 s2 = (im2*wAll).sum()/nPix
758 p = (im1TimesW*im2).sum()/nPix
764 def _initialParsForPolynomial(order):
766 pars = np.zeros(order, dtype=np.float)
773 def _boundsForPolynomial(initialPars):
774 lowers = [np.NINF
for p
in initialPars]
775 uppers = [np.inf
for p
in initialPars]
777 return (lowers, uppers)
780 def _boundsForAstier(initialPars):
781 lowers = [np.NINF
for p
in initialPars]
782 uppers = [np.inf
for p
in initialPars]
783 return (lowers, uppers)
786 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
787 """Return a boolean array to mask bad points.
789 A linear function has a constant ratio, so find the median
790 value of the ratios, and exclude the points that deviate
791 from that by more than a factor of maxDeviationPositive/negative.
792 Asymmetric deviations are supported as we expect the PTC to turn
793 down as the flux increases, but sometimes it anomalously turns
794 upwards just before turning over, which ruins the fits, so it
795 is wise to be stricter about restricting positive outliers than
798 Too high and points that are so bad that fit will fail will be included
799 Too low and the non-linear points will be excluded, biasing the NL fit."""
800 ratios = [b/a
for (a, b)
in zip(means, variances)]
801 medianRatio = np.median(ratios)
802 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
805 maxDeviationPositive = abs(maxDeviationPositive)
806 maxDeviationNegative = -1. * abs(maxDeviationNegative)
808 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
809 else False for r
in ratioDeviations])
812 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
814 nBad = Counter(array)[0]
819 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
822 array[array == 0] = substituteValue
826 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
828 Fit the photon transfer curve with either a polynomial of the order
829 specified in the task config, or using the Astier approximation.
831 Sigma clipping is performed iteratively for the fit, as well as an
832 initial clipping of data points that are more than
833 config.initialNonLinearityExclusionThreshold away from lying on a
834 straight line. This other step is necessary because the photon transfer
835 curve turns over catastrophically at very high flux (because saturation
836 drops the variance to ~0) and these far outliers cause the initial fit
837 to fail, meaning the sigma cannot be calculated to perform the
842 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
843 The dataset containing the means, variances and exposure times
846 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
847 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
851 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
852 This is the same dataset as the input paramter, however, it has been modified
853 to include information such as the fit vectors and the fit parameters. See
854 the class `PhotonTransferCurveDatase`.
857 matrixSide = self.config.maximumRangeCovariancesAstier
858 nanMatrix = np.empty((matrixSide, matrixSide))
859 nanMatrix[:] = np.nan
861 for amp
in dataset.ampNames:
862 lenInputTimes = len(dataset.rawExpTimes[amp])
863 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
864 listNanMatrix[:] = np.nan
866 dataset.covariances[amp] = listNanMatrix
867 dataset.covariancesModel[amp] = listNanMatrix
868 dataset.covariancesSqrtWeights[amp] = listNanMatrix
869 dataset.aMatrix[amp] = nanMatrix
870 dataset.bMatrix[amp] = nanMatrix
871 dataset.covariancesNoB[amp] = listNanMatrix
872 dataset.covariancesModelNoB[amp] = listNanMatrix
873 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
874 dataset.aMatrixNoB[amp] = nanMatrix
876 def errFunc(p, x, y):
877 return ptcFunc(p, x) - y
879 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
880 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
882 for i, ampName
in enumerate(dataset.ampNames):
883 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
884 meanVecOriginal = np.array(dataset.rawMeans[ampName])
885 varVecOriginal = np.array(dataset.rawVars[ampName])
888 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
889 (meanVecOriginal <= self.config.maxMeanSignal))
892 self.config.initialNonLinearityExclusionThresholdPositive,
893 self.config.initialNonLinearityExclusionThresholdNegative)
894 if not (mask.any()
and goodPoints.any()):
895 msg = (f
"\nSERIOUS: All points in either mask: {mask} or goodPoints: {goodPoints} are bad."
896 f
"Setting {ampName} to BAD.")
900 dataset.badAmps.append(ampName)
901 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
902 dataset.gain[ampName] = np.nan
903 dataset.gainErr[ampName] = np.nan
904 dataset.noise[ampName] = np.nan
905 dataset.noiseErr[ampName] = np.nan
906 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
907 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
908 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
909 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
910 dataset.ptcFitChiSq[ampName] = np.nan
911 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
912 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
913 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
916 mask = mask & goodPoints
918 if ptcFitType ==
'EXPAPPROXIMATION':
920 parsIniPtc = [-1e-9, 1.0, 10.]
922 if ptcFitType ==
'POLYNOMIAL':
923 ptcFunc = funcPolynomial
929 while count <= maxIterationsPtcOutliers:
933 meanTempVec = meanVecOriginal[mask]
934 varTempVec = varVecOriginal[mask]
935 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
941 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
942 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
943 mask = mask & newMask
944 if not (mask.any()
and newMask.any()):
945 msg = (f
"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
946 f
"Setting {ampName} to BAD.")
950 dataset.badAmps.append(ampName)
951 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
952 dataset.gain[ampName] = np.nan
953 dataset.gainErr[ampName] = np.nan
954 dataset.noise[ampName] = np.nan
955 dataset.noiseErr[ampName] = np.nan
956 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
957 if ptcFitType
in [
"POLYNOMIAL", ]
else
958 np.repeat(np.nan, 3))
959 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
960 if ptcFitType
in [
"POLYNOMIAL", ]
else
961 np.repeat(np.nan, 3))
962 dataset.ptcFitChiSq[ampName] = np.nan
963 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
964 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
965 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
967 nDroppedTotal = Counter(mask)[
False]
968 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
971 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
973 if not (mask.any()
and newMask.any()):
975 dataset.expIdMask[ampName] = mask
977 meanVecFinal = meanVecOriginal[mask]
978 varVecFinal = varVecOriginal[mask]
980 if Counter(mask)[
False] > 0:
981 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
982 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
984 if (len(meanVecFinal) < len(parsIniPtc)):
985 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
986 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
990 dataset.badAmps.append(ampName)
991 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
992 dataset.gain[ampName] = np.nan
993 dataset.gainErr[ampName] = np.nan
994 dataset.noise[ampName] = np.nan
995 dataset.noiseErr[ampName] = np.nan
996 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
997 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
998 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
999 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
1000 dataset.ptcFitChiSq[ampName] = np.nan
1001 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1002 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1003 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1007 if self.config.doFitBootstrap:
1008 parsFit, parsFitErr, reducedChiSqPtc =
fitBootstrap(parsIniPtc, meanVecFinal,
1009 varVecFinal, ptcFunc,
1010 weightsY=1./np.sqrt(varVecFinal))
1012 parsFit, parsFitErr, reducedChiSqPtc =
fitLeastSq(parsIniPtc, meanVecFinal,
1013 varVecFinal, ptcFunc,
1014 weightsY=1./np.sqrt(varVecFinal))
1015 dataset.ptcFitPars[ampName] = parsFit
1016 dataset.ptcFitParsError[ampName] = parsFitErr
1017 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1020 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1021 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength),
'constant',
1022 constant_values=np.nan)
1023 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1024 'constant', constant_values=np.nan)
1025 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength),
'constant',
1026 constant_values=np.nan)
1028 if ptcFitType ==
'EXPAPPROXIMATION':
1029 ptcGain = parsFit[1]
1030 ptcGainErr = parsFitErr[1]
1031 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1032 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1033 if ptcFitType ==
'POLYNOMIAL':
1034 ptcGain = 1./parsFit[1]
1035 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1036 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1037 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1038 dataset.gain[ampName] = ptcGain
1039 dataset.gainErr[ampName] = ptcGainErr
1040 dataset.noise[ampName] = ptcNoise
1041 dataset.noiseErr[ampName] = ptcNoiseErr
1042 if not len(dataset.ptcFitType) == 0:
1043 dataset.ptcFitType = ptcFitType
1044 if len(dataset.badAmps) == 0:
1045 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))