23 import matplotlib.pyplot
as plt
24 from collections
import Counter
29 from .utils
import (fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
30 from scipy.optimize
import least_squares
34 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect, fitData)
35 from .linearity
import LinearitySolveTask
36 from .photodiode
import getBOTphotodiodeData
38 from lsst.pipe.tasks.getRepositoryData
import DataRefListRunner
43 __all__ = [
'MeasurePhotonTransferCurveTask',
44 'MeasurePhotonTransferCurveTaskConfig']
48 """Config class for photon transfer curve measurement task"""
49 ccdKey = pexConfig.Field(
51 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
54 ptcFitType = pexConfig.ChoiceField(
56 doc=
"Fit PTC to Eq. 16, Eq. 20 in Astier+19, or to a polynomial.",
59 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
60 "EXPAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16).",
61 "FULLCOVARIANCE":
"Full covariances model in Astier+19 (Eq. 20)"
64 maximumRangeCovariancesAstier = pexConfig.Field(
66 doc=
"Maximum range of covariances as in Astier+19",
69 covAstierRealSpace = pexConfig.Field(
71 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
74 polynomialFitDegree = pexConfig.Field(
76 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
79 linearity = pexConfig.ConfigurableField(
80 target=LinearitySolveTask,
81 doc=
"Task to solve the linearity."
84 doCreateLinearizer = pexConfig.Field(
86 doc=
"Calculate non-linearity and persist linearizer?",
90 binSize = pexConfig.Field(
92 doc=
"Bin the image by this factor in both dimensions.",
95 minMeanSignal = pexConfig.DictField(
98 doc=
"Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
99 " The same cut is applied to all amps if this dictionary is of the form"
100 " {'ALL_AMPS': value}",
101 default={
'ALL_AMPS': 0.0},
103 maxMeanSignal = pexConfig.DictField(
106 doc=
"Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
107 " The same cut is applied to all amps if this dictionary is of the form"
108 " {'ALL_AMPS': value}",
109 default={
'ALL_AMPS': 1e6},
111 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
113 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
114 " linear in the positive direction, from the PTC fit. Note that these points will also be"
115 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
116 " to allow an accurate determination of the sigmas for said iterative fit.",
121 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
123 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
124 " linear in the negative direction, from the PTC fit. Note that these points will also be"
125 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
126 " to allow an accurate determination of the sigmas for said iterative fit.",
131 sigmaCutPtcOutliers = pexConfig.Field(
133 doc=
"Sigma cut for outlier rejection in PTC.",
136 maskNameList = pexConfig.ListField(
138 doc=
"Mask list to exclude from statistics calculations.",
139 default=[
'SUSPECT',
'BAD',
'NO_DATA'],
141 nSigmaClipPtc = pexConfig.Field(
143 doc=
"Sigma cut for afwMath.StatisticsControl()",
146 nIterSigmaClipPtc = pexConfig.Field(
148 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
151 minNumberGoodPixelsForFft = pexConfig.Field(
153 doc=
"Minimum number of acceptable good pixels per amp to calculate the covariances via FFT.",
156 maxIterationsPtcOutliers = pexConfig.Field(
158 doc=
"Maximum number of iterations for outlier rejection in PTC.",
161 doFitBootstrap = pexConfig.Field(
163 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
166 doPhotodiode = pexConfig.Field(
168 doc=
"Apply a correction based on the photodiode readings if available?",
171 photodiodeDataPath = pexConfig.Field(
173 doc=
"Gen2 only: path to locate the data photodiode data files.",
176 instrumentName = pexConfig.Field(
178 doc=
"Instrument name.",
184 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
186 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
187 used in astronomical detectors characterization (e.g., Janesick 2001,
188 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
189 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
190 times. The difference image of each pair is formed to eliminate fixed pattern noise,
191 and then the variance of the difference image and the mean of the average image
192 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
193 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
194 arXiv:1905.08677) can be fitted to the PTC curve. These models include
195 parameters such as the gain (e/DN) and readout noise.
197 Linearizers to correct for signal-chain non-linearity are also calculated.
198 The `Linearizer` class, in general, can support per-amp linearizers, but in this
199 task this is not supported.
201 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
202 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
203 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
210 Positional arguments passed to the Task constructor. None used at this
213 Keyword arguments passed on to the Task constructor. None used at this
218 RunnerClass = DataRefListRunner
219 ConfigClass = MeasurePhotonTransferCurveTaskConfig
220 _DefaultName =
"measurePhotonTransferCurve"
223 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
224 self.makeSubtask(
"linearity")
225 plt.interactive(
False)
226 self.config.validate()
231 """Run the Photon Transfer Curve (PTC) measurement task.
233 For a dataRef (which is each detector here),
234 and given a list of exposure pairs (postISR) at different exposure times,
239 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
240 Data references for exposures for detectors to process.
242 if len(dataRefList) < 2:
243 raise RuntimeError(
"Insufficient inputs to combine.")
246 dataRef = dataRefList[0]
248 detNum = dataRef.dataId[self.config.ccdKey]
249 camera = dataRef.get(
'camera')
250 detector = camera[dataRef.dataId[self.config.ccdKey]]
252 amps = detector.getAmplifiers()
253 ampNames = [amp.getName()
for amp
in amps]
254 datasetPtc = PhotonTransferCurveDataset(ampNames, self.config.ptcFitType)
259 for (exp1, exp2)
in expPairs.values():
260 id1 = exp1.getInfo().getVisitInfo().getExposureId()
261 id2 = exp2.getInfo().getVisitInfo().getExposureId()
262 expIds.append((id1, id2))
263 self.log.info(f
"Measuring PTC using {expIds} exposures for detector {detector.getId()}")
268 if self.config.doPhotodiode:
269 for (expId1, expId2)
in expIds:
271 for i, expId
in enumerate([expId1, expId2]):
276 dataRef.dataId[
'expId'] = expId//1000
277 if self.config.photodiodeDataPath:
282 charges[i] = photodiodeData.getCharge()
286 self.log.warn(f
"No photodiode data found for {expId}")
288 for ampName
in ampNames:
289 datasetPtc.photoCharge[ampName].append((charges[0], charges[1]))
293 for ampName
in ampNames:
294 datasetPtc.photoCharge[ampName] = np.repeat(np.nan, len(expIds))
296 for ampName
in ampNames:
297 datasetPtc.inputExpIdPairs[ampName] = expIds
299 maxMeanSignalDict = {ampName: 1e6
for ampName
in ampNames}
300 minMeanSignalDict = {ampName: 0.0
for ampName
in ampNames}
301 for ampName
in ampNames:
302 if 'ALL_AMPS' in self.config.maxMeanSignal:
303 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[
'ALL_AMPS']
304 elif ampName
in self.config.maxMeanSignal:
305 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
307 if 'ALL_AMPS' in self.config.minMeanSignal:
308 minMeanSignalDict[ampName] = self.config.minMeanSignal[
'ALL_AMPS']
309 elif ampName
in self.config.minMeanSignal:
310 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
314 for expTime, (exp1, exp2)
in expPairs.items():
315 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
316 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
319 tags = [
'mu',
'i',
'j',
'var',
'cov',
'npix',
'ext',
'expTime',
'ampName']
320 for ampNumber, amp
in enumerate(detector):
321 ampName = amp.getName()
323 doRealSpace = self.config.covAstierRealSpace
324 muDiff, varDiff, covAstier = self.
measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
325 covAstierRealSpace=doRealSpace)
327 if np.isnan(muDiff)
or np.isnan(varDiff)
or (covAstier
is None):
328 msg = (f
"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
329 f
" {expId2} of detector {detNum}.")
333 if (muDiff <= minMeanSignalDict[ampName])
or (muDiff >= maxMeanSignalDict[ampName]):
336 datasetPtc.rawExpTimes[ampName].append(expTime)
337 datasetPtc.rawMeans[ampName].append(muDiff)
338 datasetPtc.rawVars[ampName].append(varDiff)
340 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName)
for covRow
in covAstier]
341 if nAmpsNan == len(ampNames):
342 msg = f
"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
346 tupleRecords += tupleRows
347 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
349 for ampName
in datasetPtc.ampNames:
351 index = np.argsort(datasetPtc.rawMeans[ampName])
352 datasetPtc.rawExpTimes[ampName] = np.array(datasetPtc.rawExpTimes[ampName])[index]
353 datasetPtc.rawMeans[ampName] = np.array(datasetPtc.rawMeans[ampName])[index]
354 datasetPtc.rawVars[ampName] = np.array(datasetPtc.rawVars[ampName])[index]
356 if self.config.ptcFitType
in [
"FULLCOVARIANCE", ]:
360 newDatasetPtc = copy.copy(datasetPtc)
361 newDatasetPtc = self.
fitPtc(newDatasetPtc,
'EXPAPPROXIMATION')
362 for ampName
in datasetPtc.ampNames:
363 datasetPtc.expIdMask[ampName] = newDatasetPtc.expIdMask[ampName]
365 datasetPtc.fitType =
"FULLCOVARIANCE"
367 elif self.config.ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
370 datasetPtc = self.
fitPtc(datasetPtc, self.config.ptcFitType)
372 detName = detector.getName()
373 now = datetime.datetime.utcnow()
374 calibDate = now.strftime(
"%Y-%m-%d")
375 butler = dataRef.getButler()
377 datasetPtc.updateMetadata(setDate=
True, camera=camera, detector=detector)
380 if self.config.doCreateLinearizer:
386 dimensions = {
'camera': camera.getName(),
'detector': detector.getId()}
387 linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
388 linearizer = linearityResults.outputLinearizer
390 self.log.info(
"Writing linearizer:")
392 detName = detector.getName()
393 now = datetime.datetime.utcnow()
394 calibDate = now.strftime(
"%Y-%m-%d")
396 butler.put(linearizer, datasetType=
'linearizer',
397 dataId={
'detector': detNum,
'detectorName': detName,
'calibDate': calibDate})
399 self.log.info(f
"Writing PTC data.")
400 butler.put(datasetPtc, datasetType=
'photonTransferCurveDataset', dataId={
'detector': detNum,
401 'detectorName': detName,
'calibDate': calibDate})
403 return pipeBase.Struct(exitStatus=0)
406 """Produce a list of flat pairs indexed by exposure time.
410 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
411 Data references for exposures for detectors to process.
415 flatPairs : `dict` [`float`, `lsst.afw.image.exposure.exposure.ExposureF`]
416 Dictionary that groups flat-field exposures that have the same exposure time (seconds).
420 We use the difference of one pair of flat-field images taken at the same exposure time when
421 calculating the PTC to reduce Fixed Pattern Noise. If there are > 2 flat-field images with the
422 same exposure time, the first two are kept and the rest discarded.
427 for dataRef
in dataRefList:
429 tempFlat = dataRef.get(
"postISRCCD")
431 self.log.warn(
"postISR exposure could not be retrieved. Ignoring flat.")
433 expDate = tempFlat.getInfo().getVisitInfo().getDate().get()
434 expDict.setdefault(expDate, tempFlat)
435 sortedExps = {k: expDict[k]
for k
in sorted(expDict)}
438 for exp
in sortedExps:
439 tempFlat = sortedExps[exp]
440 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime()
441 listAtExpTime = flatPairs.setdefault(expTime, [])
442 if len(listAtExpTime) >= 2:
443 self.log.warn(f
"Already found 2 exposures at expTime {expTime}. "
444 f
"Ignoring exposure {tempFlat.getInfo().getVisitInfo().getExposureId()}")
446 listAtExpTime.append(tempFlat)
449 for (key, value)
in flatPairs.items():
451 keysToDrop.append(key)
454 for key
in keysToDrop:
455 self.log.warn(f
"Only one exposure found at expTime {key}. Dropping exposure "
456 f
"{flatPairs[key][0].getInfo().getVisitInfo().getExposureId()}.")
458 sortedFlatPairs = {k: flatPairs[k]
for k
in sorted(flatPairs)}
459 return sortedFlatPairs
462 """Fit measured flat covariances to full model in Astier+19.
466 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
467 The dataset containing information such as the means, variances and exposure times.
469 covariancesWithTagsArray : `numpy.recarray`
470 Tuple with at least (mu, cov, var, i, j, npix), where:
471 mu : 0.5*(m1 + m2), where:
472 mu1: mean value of flat1
473 mu2: mean value of flat2
474 cov: covariance value at lag(i, j)
475 var: variance(covariance value at lag(0, 0))
478 npix: number of pixels used for covariance calculation.
482 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
483 This is the same dataset as the input paramter, however, it has been modified
484 to include information such as the fit vectors and the fit parameters. See
485 the class `PhotonTransferCurveDatase`.
488 covFits, covFitsNoB =
fitData(covariancesWithTagsArray,
489 r=self.config.maximumRangeCovariancesAstier,
490 expIdMask=dataset.expIdMask)
495 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
499 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
500 The dataset containing information such as the means, variances and exposure times.
503 Dictionary of CovFit objects, with amp names as keys.
506 Dictionary of CovFit objects, with amp names as keys, and 'b=0' in Eq. 20 of Astier+19.
510 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
511 This is the same dataset as the input paramter, however, it has been modified
512 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
513 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
514 See the class `PhotonTransferCurveDatase`.
516 assert(len(covFits) == len(covFitsNoB))
518 for i, amp
in enumerate(dataset.ampNames):
519 lenInputTimes = len(dataset.rawExpTimes[amp])
521 dataset.ptcFitPars[amp] = np.nan
522 dataset.ptcFitParsError[amp] = np.nan
523 dataset.ptcFitChiSq[amp] = np.nan
524 if (amp
in covFits
and (covFits[amp].covParams
is not None)
and
525 (covFitsNoB[amp].covParams
is not None)):
527 fitNoB = covFitsNoB[amp]
529 dataset.covariances[amp] = fit.cov
530 dataset.covariancesModel[amp] = fit.evalCovModel()
531 dataset.covariancesSqrtWeights[amp] = fit.sqrtW
532 dataset.aMatrix[amp] = fit.getA()
533 dataset.bMatrix[amp] = fit.getB()
534 dataset.covariancesNoB[amp] = fitNoB.cov
535 dataset.covariancesModelNoB[amp] = fitNoB.evalCovModel()
536 dataset.covariancesSqrtWeightsNoB[amp] = fitNoB.sqrtW
537 dataset.aMatrixNoB[amp] = fitNoB.getA()
539 (meanVecFinal, varVecFinal, varVecModel,
540 wc, varMask) = fit.getFitData(0, 0, divideByMu=
False)
543 dataset.gain[amp] = gain
544 dataset.gainErr[amp] = fit.getGainErr()
545 dataset.noise[amp] = np.sqrt(fit.getRon())
546 dataset.noiseErr[amp] = fit.getRonErr()
548 padLength = lenInputTimes - len(varVecFinal)
549 dataset.finalVars[amp] = np.pad(varVecFinal/(gain**2), (0, padLength),
'constant',
550 constant_values=np.nan)
551 dataset.finalModelVars[amp] = np.pad(varVecModel/(gain**2), (0, padLength),
'constant',
552 constant_values=np.nan)
553 dataset.finalMeans[amp] = np.pad(meanVecFinal/gain, (0, padLength),
'constant',
554 constant_values=np.nan)
558 matrixSide = self.config.maximumRangeCovariancesAstier
559 nanMatrix = np.full((matrixSide, matrixSide), np.nan)
560 listNanMatrix = np.full((lenInputTimes, matrixSide, matrixSide), np.nan)
562 dataset.covariances[amp] = listNanMatrix
563 dataset.covariancesModel[amp] = listNanMatrix
564 dataset.covariancesSqrtWeights[amp] = listNanMatrix
565 dataset.aMatrix[amp] = nanMatrix
566 dataset.bMatrix[amp] = nanMatrix
567 dataset.covariancesNoB[amp] = listNanMatrix
568 dataset.covariancesModelNoB[amp] = listNanMatrix
569 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
570 dataset.aMatrixNoB[amp] = nanMatrix
572 dataset.expIdMask[amp] = np.repeat(np.nan, lenInputTimes)
573 dataset.gain[amp] = np.nan
574 dataset.gainErr[amp] = np.nan
575 dataset.noise[amp] = np.nan
576 dataset.noiseErr[amp] = np.nan
577 dataset.finalVars[amp] = np.repeat(np.nan, lenInputTimes)
578 dataset.finalModelVars[amp] = np.repeat(np.nan, lenInputTimes)
579 dataset.finalMeans[amp] = np.repeat(np.nan, lenInputTimes)
584 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
586 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
587 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
588 keep one (covariance).
592 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
593 First exposure of flat field pair.
595 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
596 Second exposure of flat field pair.
598 region : `lsst.geom.Box2I`, optional
599 Region of each exposure where to perform the calculations (e.g, an amplifier).
601 covAstierRealSpace : `bool`, optional
602 Should the covariannces in Astier+19 be calculated in real space or via FFT?
603 See Appendix A of Astier+19.
607 mu : `float` or `NaN`
608 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
609 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
611 varDiff : `float` or `NaN`
612 Half of the clipped variance of the difference of the regions inthe two input
613 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
615 covDiffAstier : `list` or `None`
616 List with tuples of the form (dx, dy, var, cov, npix), where:
622 Variance at (dx, dy).
624 Covariance at (dx, dy).
626 Number of pixel pairs used to evaluate var and cov.
627 If either mu1 or m2 are NaN's, the returned value is None.
630 if region
is not None:
631 im1Area = exposure1.maskedImage[region]
632 im2Area = exposure2.maskedImage[region]
634 im1Area = exposure1.maskedImage
635 im2Area = exposure2.maskedImage
637 if self.config.binSize > 1:
638 im1Area = afwMath.binImage(im1Area, self.config.binSize)
639 im2Area = afwMath.binImage(im2Area, self.config.binSize)
641 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
642 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
643 self.config.nIterSigmaClipPtc,
645 im1StatsCtrl.setNanSafe(
True)
646 im1StatsCtrl.setAndMask(im1MaskVal)
648 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
649 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
650 self.config.nIterSigmaClipPtc,
652 im2StatsCtrl.setNanSafe(
True)
653 im2StatsCtrl.setAndMask(im2MaskVal)
656 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
657 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
658 if np.isnan(mu1)
or np.isnan(mu2):
659 self.log.warn(f
"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
660 return np.nan, np.nan,
None
665 temp = im2Area.clone()
667 diffIm = im1Area.clone()
672 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
673 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
674 self.config.nIterSigmaClipPtc,
676 diffImStatsCtrl.setNanSafe(
True)
677 diffImStatsCtrl.setAndMask(diffImMaskVal)
679 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
682 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
683 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
686 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
689 if np.sum(w) < self.config.minNumberGoodPixelsForFft:
690 self.log.warn(f
"Number of good points for FFT ({np.sum(w)}) is less than threshold "
691 f
"({self.config.minNumberGoodPixelsForFft})")
692 return np.nan, np.nan,
None
694 maxRangeCov = self.config.maximumRangeCovariancesAstier
695 if covAstierRealSpace:
696 covDiffAstier =
computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
698 shapeDiff = diffIm.getImage().getArray().shape
699 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
700 c =
CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
701 covDiffAstier = c.reportCovFft(maxRangeCov)
703 return mu, varDiff, covDiffAstier
706 """Compute covariances of diffImage in real space.
708 For lags larger than ~25, it is slower than the FFT way.
709 Taken from https://github.com/PierreAstier/bfptc/
713 diffImage : `numpy.array`
714 Image to compute the covariance of.
716 weightImage : `numpy.array`
717 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
720 Last index of the covariance to be computed.
725 List with tuples of the form (dx, dy, var, cov, npix), where:
731 Variance at (dx, dy).
733 Covariance at (dx, dy).
735 Number of pixel pairs used to evaluate var and cov.
740 for dy
in range(maxRange + 1):
741 for dx
in range(0, maxRange + 1):
744 cov2, nPix2 = self.
covDirectValue(diffImage, weightImage, dx, -dy)
745 cov = 0.5*(cov1 + cov2)
749 if (dx == 0
and dy == 0):
751 outList.append((dx, dy, var, cov, nPix))
756 """Compute covariances of diffImage in real space at lag (dx, dy).
758 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
762 diffImage : `numpy.array`
763 Image to compute the covariance of.
765 weightImage : `numpy.array`
766 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
777 Covariance at (dx, dy)
780 Number of pixel pairs used to evaluate var and cov.
782 (nCols, nRows) = diffImage.shape
786 (dx, dy) = (-dx, -dy)
790 im1 = diffImage[dy:, dx:]
791 w1 = weightImage[dy:, dx:]
792 im2 = diffImage[:nCols - dy, :nRows - dx]
793 w2 = weightImage[:nCols - dy, :nRows - dx]
795 im1 = diffImage[:nCols + dy, dx:]
796 w1 = weightImage[:nCols + dy, dx:]
797 im2 = diffImage[-dy:, :nRows - dx]
798 w2 = weightImage[-dy:, :nRows - dx]
804 s1 = im1TimesW.sum()/nPix
805 s2 = (im2*wAll).sum()/nPix
806 p = (im1TimesW*im2).sum()/nPix
812 def _initialParsForPolynomial(order):
814 pars = np.zeros(order, dtype=np.float)
821 def _boundsForPolynomial(initialPars, lowers=[], uppers=[]):
823 lowers = [np.NINF
for p
in initialPars]
825 uppers = [np.inf
for p
in initialPars]
827 return (lowers, uppers)
830 def _boundsForAstier(initialPars, lowers=[], uppers=[]):
832 lowers = [np.NINF
for p
in initialPars]
834 uppers = [np.inf
for p
in initialPars]
835 return (lowers, uppers)
838 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
839 """Return a boolean array to mask bad points.
843 means : `numpy.array`
844 Input array with mean signal values.
846 variances : `numpy.array`
847 Input array with variances at each mean value.
849 maxDeviationPositive : `float`
850 Maximum deviation from being constant for the variance/mean
851 ratio, in the positive direction.
853 maxDeviationNegative : `float`
854 Maximum deviation from being constant for the variance/mean
855 ratio, in the negative direction.
859 goodPoints : `numpy.array` [`bool`]
860 Boolean array to select good (`True`) and bad (`False`)
865 A linear function has a constant ratio, so find the median
866 value of the ratios, and exclude the points that deviate
867 from that by more than a factor of maxDeviationPositive/negative.
868 Asymmetric deviations are supported as we expect the PTC to turn
869 down as the flux increases, but sometimes it anomalously turns
870 upwards just before turning over, which ruins the fits, so it
871 is wise to be stricter about restricting positive outliers than
874 Too high and points that are so bad that fit will fail will be included
875 Too low and the non-linear points will be excluded, biasing the NL fit.
877 This function also masks points after the variance starts decreasing.
880 assert(len(means) == len(variances))
881 ratios = [b/a
for (a, b)
in zip(means, variances)]
882 medianRatio = np.nanmedian(ratios)
883 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
886 maxDeviationPositive = abs(maxDeviationPositive)
887 maxDeviationNegative = -1. * abs(maxDeviationNegative)
889 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
890 else False for r
in ratioDeviations])
893 pivot = np.where(np.array(np.diff(variances)) < 0)[0]
897 pivot = np.min(pivot)
898 goodPoints[pivot:len(goodPoints)] =
False
901 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
903 nBad = Counter(array)[0]
908 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
911 array[array == 0] = substituteValue
915 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
917 Fit the photon transfer curve with either a polynomial of the order
918 specified in the task config, or using the Astier approximation.
920 Sigma clipping is performed iteratively for the fit, as well as an
921 initial clipping of data points that are more than
922 config.initialNonLinearityExclusionThreshold away from lying on a
923 straight line. This other step is necessary because the photon transfer
924 curve turns over catastrophically at very high flux (because saturation
925 drops the variance to ~0) and these far outliers cause the initial fit
926 to fail, meaning the sigma cannot be calculated to perform the
931 dataset : `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
932 The dataset containing the means, variances and exposure times
935 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
936 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
940 dataset: `lsst.ip.isr.ptcDataset.PhotonTransferCurveDataset`
941 This is the same dataset as the input paramter, however, it has been modified
942 to include information such as the fit vectors and the fit parameters. See
943 the class `PhotonTransferCurveDatase`.
946 matrixSide = self.config.maximumRangeCovariancesAstier
947 nanMatrix = np.empty((matrixSide, matrixSide))
948 nanMatrix[:] = np.nan
950 for amp
in dataset.ampNames:
951 lenInputTimes = len(dataset.rawExpTimes[amp])
952 listNanMatrix = np.empty((lenInputTimes, matrixSide, matrixSide))
953 listNanMatrix[:] = np.nan
955 dataset.covariances[amp] = listNanMatrix
956 dataset.covariancesModel[amp] = listNanMatrix
957 dataset.covariancesSqrtWeights[amp] = listNanMatrix
958 dataset.aMatrix[amp] = nanMatrix
959 dataset.bMatrix[amp] = nanMatrix
960 dataset.covariancesNoB[amp] = listNanMatrix
961 dataset.covariancesModelNoB[amp] = listNanMatrix
962 dataset.covariancesSqrtWeightsNoB[amp] = listNanMatrix
963 dataset.aMatrixNoB[amp] = nanMatrix
965 def errFunc(p, x, y):
966 return ptcFunc(p, x) - y
968 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
969 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
971 for i, ampName
in enumerate(dataset.ampNames):
972 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
973 meanVecOriginal = np.array(dataset.rawMeans[ampName])
974 varVecOriginal = np.array(dataset.rawVars[ampName])
978 self.config.initialNonLinearityExclusionThresholdPositive,
979 self.config.initialNonLinearityExclusionThresholdNegative)
980 if not (goodPoints.any()):
981 msg = (f
"\nSERIOUS: All points in goodPoints: {goodPoints} are bad."
982 f
"Setting {ampName} to BAD.")
986 dataset.badAmps.append(ampName)
987 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
988 dataset.gain[ampName] = np.nan
989 dataset.gainErr[ampName] = np.nan
990 dataset.noise[ampName] = np.nan
991 dataset.noiseErr[ampName] = np.nan
992 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
993 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
994 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
995 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
996 dataset.ptcFitChiSq[ampName] = np.nan
997 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
998 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
999 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1004 if ptcFitType ==
'EXPAPPROXIMATION':
1005 ptcFunc = funcAstier
1006 parsIniPtc = [-1e-9, 1.0, 10.]
1009 uppers=[1e-4, 2.5, 2000])
1010 if ptcFitType ==
'POLYNOMIAL':
1011 ptcFunc = funcPolynomial
1017 while count <= maxIterationsPtcOutliers:
1021 meanTempVec = meanVecOriginal[mask]
1022 varTempVec = varVecOriginal[mask]
1023 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1029 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1030 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
1031 mask = mask & newMask
1032 if not (mask.any()
and newMask.any()):
1033 msg = (f
"\nSERIOUS: All points in either mask: {mask} or newMask: {newMask} are bad. "
1034 f
"Setting {ampName} to BAD.")
1038 dataset.badAmps.append(ampName)
1039 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1040 dataset.gain[ampName] = np.nan
1041 dataset.gainErr[ampName] = np.nan
1042 dataset.noise[ampName] = np.nan
1043 dataset.noiseErr[ampName] = np.nan
1044 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1045 if ptcFitType
in [
"POLYNOMIAL", ]
else
1046 np.repeat(np.nan, 3))
1047 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
1048 if ptcFitType
in [
"POLYNOMIAL", ]
else
1049 np.repeat(np.nan, 3))
1050 dataset.ptcFitChiSq[ampName] = np.nan
1051 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1052 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1053 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1055 nDroppedTotal = Counter(mask)[
False]
1056 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1059 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1061 if not (mask.any()
and newMask.any()):
1063 dataset.expIdMask[ampName] = mask
1065 meanVecFinal = meanVecOriginal[mask]
1066 varVecFinal = varVecOriginal[mask]
1068 if Counter(mask)[
False] > 0:
1069 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
1070 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1072 if (len(meanVecFinal) < len(parsIniPtc)):
1073 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1074 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1078 dataset.badAmps.append(ampName)
1079 dataset.expIdMask[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1080 dataset.gain[ampName] = np.nan
1081 dataset.gainErr[ampName] = np.nan
1082 dataset.noise[ampName] = np.nan
1083 dataset.noiseErr[ampName] = np.nan
1084 dataset.ptcFitPars[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
1085 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
1086 dataset.ptcFitParsError[ampName] = (np.repeat(np.nan, self.config.polynomialFitDegree + 1)
if
1087 ptcFitType
in [
"POLYNOMIAL", ]
else np.repeat(np.nan, 3))
1088 dataset.ptcFitChiSq[ampName] = np.nan
1089 dataset.finalVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1090 dataset.finalModelVars[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1091 dataset.finalMeans[ampName] = np.repeat(np.nan, len(dataset.rawExpTimes[ampName]))
1095 if self.config.doFitBootstrap:
1096 parsFit, parsFitErr, reducedChiSqPtc =
fitBootstrap(parsIniPtc, meanVecFinal,
1097 varVecFinal, ptcFunc,
1098 weightsY=1./np.sqrt(varVecFinal))
1100 parsFit, parsFitErr, reducedChiSqPtc =
fitLeastSq(parsIniPtc, meanVecFinal,
1101 varVecFinal, ptcFunc,
1102 weightsY=1./np.sqrt(varVecFinal))
1103 dataset.ptcFitPars[ampName] = parsFit
1104 dataset.ptcFitParsError[ampName] = parsFitErr
1105 dataset.ptcFitChiSq[ampName] = reducedChiSqPtc
1108 padLength = len(dataset.rawExpTimes[ampName]) - len(varVecFinal)
1109 dataset.finalVars[ampName] = np.pad(varVecFinal, (0, padLength),
'constant',
1110 constant_values=np.nan)
1111 dataset.finalModelVars[ampName] = np.pad(ptcFunc(parsFit, meanVecFinal), (0, padLength),
1112 'constant', constant_values=np.nan)
1113 dataset.finalMeans[ampName] = np.pad(meanVecFinal, (0, padLength),
'constant',
1114 constant_values=np.nan)
1116 if ptcFitType ==
'EXPAPPROXIMATION':
1117 ptcGain = parsFit[1]
1118 ptcGainErr = parsFitErr[1]
1119 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1120 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1121 if ptcFitType ==
'POLYNOMIAL':
1122 ptcGain = 1./parsFit[1]
1123 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1124 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1125 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1126 dataset.gain[ampName] = ptcGain
1127 dataset.gainErr[ampName] = ptcGainErr
1128 dataset.noise[ampName] = ptcNoise
1129 dataset.noiseErr[ampName] = ptcNoiseErr
1130 if not len(dataset.ptcFitType) == 0:
1131 dataset.ptcFitType = ptcFitType
1132 if len(dataset.badAmps) == 0:
1133 dataset.badAmps = np.repeat(np.nan, len(list(dataset.rawExpTimes.values())[0]))