23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
29 from sqlite3
import OperationalError
30 from collections
import Counter
35 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
36 checkExpLengthEqual, fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
37 from scipy.optimize
import least_squares
41 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect, fitData)
42 from .linearity
import LinearitySolveTask
46 """Config class for photon transfer curve measurement task"""
47 ccdKey = pexConfig.Field(
49 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
52 ptcFitType = pexConfig.ChoiceField(
54 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
57 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
58 "EXPAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16).",
59 "FULLCOVARIANCE":
"Full covariances model in Astier+19 (Eq. 20)"
62 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
64 doc=
"sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
67 maxIterFullFitCovariancesAstier = pexConfig.Field(
69 doc=
"Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
72 maximumRangeCovariancesAstier = pexConfig.Field(
74 doc=
"Maximum range of covariances as in Astier+19",
77 covAstierRealSpace = pexConfig.Field(
79 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
82 polynomialFitDegree = pexConfig.Field(
84 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
87 linearity = pexConfig.ConfigurableField(
88 target=LinearitySolveTask,
89 doc=
"Task to solve the linearity."
92 doCreateLinearizer = pexConfig.Field(
94 doc=
"Calculate non-linearity and persist linearizer?",
98 binSize = pexConfig.Field(
100 doc=
"Bin the image by this factor in both dimensions.",
103 minMeanSignal = pexConfig.Field(
105 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
108 maxMeanSignal = pexConfig.Field(
110 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
113 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
115 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
116 " linear in the positive direction, from the PTC fit. Note that these points will also be"
117 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
118 " to allow an accurate determination of the sigmas for said iterative fit.",
123 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
125 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
126 " linear in the negative direction, from the PTC fit. Note that these points will also be"
127 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
128 " to allow an accurate determination of the sigmas for said iterative fit.",
133 sigmaCutPtcOutliers = pexConfig.Field(
135 doc=
"Sigma cut for outlier rejection in PTC.",
138 maskNameList = pexConfig.ListField(
140 doc=
"Mask list to exclude from statistics calculations.",
141 default=[
'SUSPECT',
'BAD',
'NO_DATA'],
143 nSigmaClipPtc = pexConfig.Field(
145 doc=
"Sigma cut for afwMath.StatisticsControl()",
148 nIterSigmaClipPtc = pexConfig.Field(
150 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
153 maxIterationsPtcOutliers = pexConfig.Field(
155 doc=
"Maximum number of iterations for outlier rejection in PTC.",
158 doFitBootstrap = pexConfig.Field(
160 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
163 instrumentName = pexConfig.Field(
165 doc=
"Instrument name.",
171 """A simple class to hold the output data from the PTC task.
173 The dataset is made up of a dictionary for each item, keyed by the
174 amplifiers' names, which much be supplied at construction time.
176 New items cannot be added to the class to save accidentally saving to the
177 wrong property, and the class can be frozen if desired.
179 inputVisitPairs records the visits used to produce the data.
180 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which is by definition
181 always the same length as inputVisitPairs, rawExpTimes, rawMeans
182 and rawVars, and is a list of bools, which are incrementally set to False
183 as points are discarded from the fits.
185 PTC fit parameters for polynomials are stored in a list in ascending order
186 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
187 with the length of the list corresponding to the order of the polynomial
193 List with the names of the amplifiers of the detector at hand.
196 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", or "FULLCOVARIANCE".
200 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
201 Output dataset from MeasurePhotonTransferCurveTask.
208 self.__dict__[
"ptcFitType"] = ptcFitType
209 self.__dict__[
"ampNames"] = ampNames
210 self.__dict__[
"badAmps"] = []
215 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
216 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
217 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
218 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
219 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
222 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
223 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
224 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
225 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
229 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
230 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
231 self.__dict__[
"ptcFitReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
238 self.__dict__[
"covariancesTuple"] = {ampName: []
for ampName
in ampNames}
239 self.__dict__[
"covariancesFitsWithNoB"] = {ampName: []
for ampName
in ampNames}
240 self.__dict__[
"covariancesFits"] = {ampName: []
for ampName
in ampNames}
241 self.__dict__[
"aMatrix"] = {ampName: []
for ampName
in ampNames}
242 self.__dict__[
"bMatrix"] = {ampName: []
for ampName
in ampNames}
245 self.__dict__[
"finalVars"] = {ampName: []
for ampName
in ampNames}
246 self.__dict__[
"finalModelVars"] = {ampName: []
for ampName
in ampNames}
247 self.__dict__[
"finalMeans"] = {ampName: []
for ampName
in ampNames}
250 """Protect class attributes"""
251 if attribute
not in self.__dict__:
252 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which"
253 " does not support setting of new attributes.")
255 self.__dict__[attribute] = value
258 """Get the visits used, i.e. not discarded, for a given amp.
260 If no mask has been created yet, all visits are returned.
262 if len(self.visitMask[ampName]) == 0:
263 return self.inputVisitPairs[ampName]
266 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
268 pairs = self.inputVisitPairs[ampName]
269 mask = self.visitMask[ampName]
271 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
274 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
278 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
280 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
281 used in astronomical detectors characterization (e.g., Janesick 2001,
282 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
283 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
284 times. The difference image of each pair is formed to eliminate fixed pattern noise,
285 and then the variance of the difference image and the mean of the average image
286 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
287 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
288 arXiv:1905.08677) can be fitted to the PTC curve. These models include
289 parameters such as the gain (e/DN) and readout noise.
291 Linearizers to correct for signal-chain non-linearity are also calculated.
292 The `Linearizer` class, in general, can support per-amp linearizers, but in this
293 task this is not supported.
295 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
296 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
297 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
304 Positional arguments passed to the Task constructor. None used at this
307 Keyword arguments passed on to the Task constructor. None used at this
312 RunnerClass = PairedVisitListTaskRunner
313 ConfigClass = MeasurePhotonTransferCurveTaskConfig
314 _DefaultName =
"measurePhotonTransferCurve"
317 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
318 self.makeSubtask(
"linearity")
319 plt.interactive(
False)
320 self.config.validate()
324 def _makeArgumentParser(cls):
325 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
327 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
328 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
329 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
330 ContainerClass=NonexistentDatasetTaskDataIdContainer,
331 help=
"The ccds to use, e.g. --id ccd=0..100")
336 """Run the Photon Transfer Curve (PTC) measurement task.
338 For a dataRef (which is each detector here),
339 and given a list of visit pairs (postISR) at different exposure times,
344 dataRef : list of lsst.daf.persistence.ButlerDataRef
345 dataRef for the detector for the visits to be fit.
347 visitPairs : `iterable` of `tuple` of `int`
348 Pairs of visit numbers to be processed together
352 detNum = dataRef.dataId[self.config.ccdKey]
353 camera = dataRef.get(
'camera')
354 detector = camera[dataRef.dataId[self.config.ccdKey]]
360 for name
in dataRef.getButler().getKeys(
'bias'):
361 if name
not in dataRef.dataId:
363 dataRef.dataId[name] = \
364 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
365 except OperationalError:
368 amps = detector.getAmplifiers()
369 ampNames = [amp.getName()
for amp
in amps]
371 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detector.getId()))
375 for (v1, v2)
in visitPairs:
378 dataRef.dataId[
'expId'] = v1
379 exp1 = dataRef.get(
"postISRCCD", immediate=
True)
380 dataRef.dataId[
'expId'] = v2
381 exp2 = dataRef.get(
"postISRCCD", immediate=
True)
383 self.log.warn(f
"postISR exposure for either expId {v1} or expId {v2} could not be retreived. "
384 "Ignoring flat pair.")
386 del dataRef.dataId[
'expId']
389 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
392 for ampNumber, amp
in enumerate(detector):
393 ampName = amp.getName()
395 doRealSpace = self.config.covAstierRealSpace
396 muDiff, varDiff, covAstier = self.
measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
397 covAstierRealSpace=doRealSpace)
398 if np.isnan(muDiff)
or np.isnan(varDiff)
or (covAstier
is None):
399 msg = (f
"NaN mean or var, or None cov in amp {ampNumber} in visit pair {v1}, {v2} "
400 "of detector {detNum}.")
404 tags = [
'mu',
'i',
'j',
'var',
'cov',
'npix',
'ext',
'expTime',
'ampName']
405 if (muDiff <= self.config.minMeanSignal)
or (muDiff >= self.config.maxMeanSignal):
407 datasetPtc.rawExpTimes[ampName].append(expTime)
408 datasetPtc.rawMeans[ampName].append(muDiff)
409 datasetPtc.rawVars[ampName].append(varDiff)
410 datasetPtc.inputVisitPairs[ampName].append((v1, v2))
412 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName)
for covRow
in covAstier]
413 if nAmpsNan == len(ampNames):
414 msg = f
"NaN mean in all amps of visit pair {v1}, {v2} of detector {detNum}."
418 tupleRecords += tupleRows
419 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
421 if self.config.ptcFitType
in [
"FULLCOVARIANCE", ]:
424 elif self.config.ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
427 datasetPtc = self.
fitPtc(datasetPtc, self.config.ptcFitType)
430 if self.config.doCreateLinearizer:
436 dimensions = {
'camera': camera.getName(),
'detector': detector.getId()}
437 linearityResults = self.linearity.run(datasetPtc, camera, dimensions)
438 linearizer = linearityResults.outputLinearizer
440 butler = dataRef.getButler()
441 self.log.info(
"Writing linearizer:")
443 detName = detector.getName()
444 now = datetime.datetime.utcnow()
445 calibDate = now.strftime(
"%Y-%m-%d")
447 butler.put(linearizer, datasetType=
'Linearizer', dataId={
'detector': detNum,
448 'detectorName': detName,
'calibDate': calibDate})
450 self.log.info(f
"Writing PTC data to {dataRef.getUri(write=True)}")
451 dataRef.put(datasetPtc, datasetType=
"photonTransferCurveDataset")
453 return pipeBase.Struct(exitStatus=0)
456 """Fit measured flat covariances to full model in Astier+19.
460 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
461 The dataset containing information such as the means, variances and exposure times.
463 covariancesWithTagsArray : `numpy.recarray`
464 Tuple with at least (mu, cov, var, i, j, npix), where:
465 mu : 0.5*(m1 + m2), where:
466 mu1: mean value of flat1
467 mu2: mean value of flat2
468 cov: covariance value at lag(i, j)
469 var: variance(covariance value at lag(0, 0))
472 npix: number of pixels used for covariance calculation.
476 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
477 This is the same dataset as the input paramter, however, it has been modified
478 to include information such as the fit vectors and the fit parameters. See
479 the class `PhotonTransferCurveDatase`.
482 covFits, covFitsNoB =
fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
483 r=self.config.maximumRangeCovariancesAstier,
484 nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
485 maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
487 dataset.covariancesTuple = covariancesWithTagsArray
488 dataset.covariancesFits = covFits
489 dataset.covariancesFitsWithNoB = covFitsNoB
495 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
499 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
500 The dataset containing information such as the means, variances and exposure times.
503 Dictionary of CovFit objects, with amp names as keys.
507 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
508 This is the same dataset as the input paramter, however, it has been modified
509 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
510 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
511 See the class `PhotonTransferCurveDatase`.
514 for i, amp
in enumerate(covFits):
516 (meanVecFinal, varVecFinal, varVecModel,
517 wc, varMask) = fit.getFitData(0, 0, divideByMu=
False, returnMasked=
True)
519 dataset.visitMask[amp] = varMask
520 dataset.gain[amp] = gain
521 dataset.gainErr[amp] = fit.getGainErr()
522 dataset.noise[amp] = np.sqrt(np.fabs(fit.getRon()))
523 dataset.noiseErr[amp] = fit.getRonErr()
524 dataset.finalVars[amp].append(varVecFinal/(gain**2))
525 dataset.finalModelVars[amp].append(varVecModel/(gain**2))
526 dataset.finalMeans[amp].append(meanVecFinal/gain)
527 dataset.aMatrix[amp].append(fit.getA())
528 dataset.bMatrix[amp].append(fit.getB())
533 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
535 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
536 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
537 keep one (covariance).
541 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
542 First exposure of flat field pair.
544 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
545 Second exposure of flat field pair.
547 region : `lsst.geom.Box2I`, optional
548 Region of each exposure where to perform the calculations (e.g, an amplifier).
550 covAstierRealSpace : `bool`, optional
551 Should the covariannces in Astier+19 be calculated in real space or via FFT?
552 See Appendix A of Astier+19.
556 mu : `float` or `NaN`
557 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
558 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
560 varDiff : `float` or `NaN`
561 Half of the clipped variance of the difference of the regions inthe two input
562 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
564 covDiffAstier : `list` or `NaN`
565 List with tuples of the form (dx, dy, var, cov, npix), where:
571 Variance at (dx, dy).
573 Covariance at (dx, dy).
575 Number of pixel pairs used to evaluate var and cov.
576 If either mu1 or m2 are NaN's, the returned value is NaN.
579 if region
is not None:
580 im1Area = exposure1.maskedImage[region]
581 im2Area = exposure2.maskedImage[region]
583 im1Area = exposure1.maskedImage
584 im2Area = exposure2.maskedImage
586 if self.config.binSize > 1:
587 im1Area = afwMath.binImage(im1Area, self.config.binSize)
588 im2Area = afwMath.binImage(im2Area, self.config.binSize)
590 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
591 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
592 self.config.nIterSigmaClipPtc,
594 im1StatsCtrl.setNanSafe(
True)
595 im1StatsCtrl.setAndMask(im1MaskVal)
597 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
598 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
599 self.config.nIterSigmaClipPtc,
601 im2StatsCtrl.setNanSafe(
True)
602 im2StatsCtrl.setAndMask(im2MaskVal)
605 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
606 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
607 if np.isnan(mu1)
or np.isnan(mu2):
608 return np.nan, np.nan,
None
613 temp = im2Area.clone()
615 diffIm = im1Area.clone()
620 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
621 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
622 self.config.nIterSigmaClipPtc,
624 diffImStatsCtrl.setNanSafe(
True)
625 diffImStatsCtrl.setAndMask(diffImMaskVal)
627 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
630 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
631 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
634 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
637 maxRangeCov = self.config.maximumRangeCovariancesAstier
638 if covAstierRealSpace:
639 covDiffAstier =
computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
641 shapeDiff = diffIm.getImage().getArray().shape
642 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
643 c =
CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
644 covDiffAstier = c.reportCovFft(maxRangeCov)
646 return mu, varDiff, covDiffAstier
649 """Compute covariances of diffImage in real space.
651 For lags larger than ~25, it is slower than the FFT way.
652 Taken from https://github.com/PierreAstier/bfptc/
656 diffImage : `numpy.array`
657 Image to compute the covariance of.
659 weightImage : `numpy.array`
660 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
663 Last index of the covariance to be computed.
668 List with tuples of the form (dx, dy, var, cov, npix), where:
674 Variance at (dx, dy).
676 Covariance at (dx, dy).
678 Number of pixel pairs used to evaluate var and cov.
683 for dy
in range(maxRange + 1):
684 for dx
in range(0, maxRange + 1):
687 cov2, nPix2 = self.
covDirectValue(diffImage, weightImage, dx, -dy)
688 cov = 0.5*(cov1 + cov2)
692 if (dx == 0
and dy == 0):
694 outList.append((dx, dy, var, cov, nPix))
699 """Compute covariances of diffImage in real space at lag (dx, dy).
701 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
705 diffImage : `numpy.array`
706 Image to compute the covariance of.
708 weightImage : `numpy.array`
709 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
720 Covariance at (dx, dy)
723 Number of pixel pairs used to evaluate var and cov.
725 (nCols, nRows) = diffImage.shape
729 (dx, dy) = (-dx, -dy)
733 im1 = diffImage[dy:, dx:]
734 w1 = weightImage[dy:, dx:]
735 im2 = diffImage[:nCols - dy, :nRows - dx]
736 w2 = weightImage[:nCols - dy, :nRows - dx]
738 im1 = diffImage[:nCols + dy, dx:]
739 w1 = weightImage[:nCols + dy, dx:]
740 im2 = diffImage[-dy:, :nRows - dx]
741 w2 = weightImage[-dy:, :nRows - dx]
747 s1 = im1TimesW.sum()/nPix
748 s2 = (im2*wAll).sum()/nPix
749 p = (im1TimesW*im2).sum()/nPix
755 def _initialParsForPolynomial(order):
757 pars = np.zeros(order, dtype=np.float)
764 def _boundsForPolynomial(initialPars):
765 lowers = [np.NINF
for p
in initialPars]
766 uppers = [np.inf
for p
in initialPars]
768 return (lowers, uppers)
771 def _boundsForAstier(initialPars):
772 lowers = [np.NINF
for p
in initialPars]
773 uppers = [np.inf
for p
in initialPars]
774 return (lowers, uppers)
777 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
778 """Return a boolean array to mask bad points.
780 A linear function has a constant ratio, so find the median
781 value of the ratios, and exclude the points that deviate
782 from that by more than a factor of maxDeviationPositive/negative.
783 Asymmetric deviations are supported as we expect the PTC to turn
784 down as the flux increases, but sometimes it anomalously turns
785 upwards just before turning over, which ruins the fits, so it
786 is wise to be stricter about restricting positive outliers than
789 Too high and points that are so bad that fit will fail will be included
790 Too low and the non-linear points will be excluded, biasing the NL fit."""
791 ratios = [b/a
for (a, b)
in zip(means, variances)]
792 medianRatio = np.median(ratios)
793 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
796 maxDeviationPositive = abs(maxDeviationPositive)
797 maxDeviationNegative = -1. * abs(maxDeviationNegative)
799 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
800 else False for r
in ratioDeviations])
803 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
805 nBad = Counter(array)[0]
810 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
813 array[array == 0] = substituteValue
817 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
819 Fit the photon transfer curve with either a polynomial of the order
820 specified in the task config, or using the Astier approximation.
822 Sigma clipping is performed iteratively for the fit, as well as an
823 initial clipping of data points that are more than
824 config.initialNonLinearityExclusionThreshold away from lying on a
825 straight line. This other step is necessary because the photon transfer
826 curve turns over catastrophically at very high flux (because saturation
827 drops the variance to ~0) and these far outliers cause the initial fit
828 to fail, meaning the sigma cannot be calculated to perform the
833 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
834 The dataset containing the means, variances and exposure times
837 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
838 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
842 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
843 This is the same dataset as the input paramter, however, it has been modified
844 to include information such as the fit vectors and the fit parameters. See
845 the class `PhotonTransferCurveDatase`.
848 def errFunc(p, x, y):
849 return ptcFunc(p, x) - y
851 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
852 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
854 for i, ampName
in enumerate(dataset.ampNames):
855 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
856 meanVecOriginal = np.array(dataset.rawMeans[ampName])
857 varVecOriginal = np.array(dataset.rawVars[ampName])
860 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
861 (meanVecOriginal <= self.config.maxMeanSignal))
864 self.config.initialNonLinearityExclusionThresholdPositive,
865 self.config.initialNonLinearityExclusionThresholdNegative)
866 mask = mask & goodPoints
868 if ptcFitType ==
'EXPAPPROXIMATION':
870 parsIniPtc = [-1e-9, 1.0, 10.]
872 if ptcFitType ==
'POLYNOMIAL':
873 ptcFunc = funcPolynomial
879 while count <= maxIterationsPtcOutliers:
883 meanTempVec = meanVecOriginal[mask]
884 varTempVec = varVecOriginal[mask]
885 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
891 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
892 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
893 mask = mask & newMask
895 nDroppedTotal = Counter(mask)[
False]
896 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
899 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
901 dataset.visitMask[ampName] = mask
903 meanVecFinal = meanVecOriginal[mask]
904 varVecFinal = varVecOriginal[mask]
906 if Counter(mask)[
False] > 0:
907 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
908 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
910 if (len(meanVecFinal) < len(parsIniPtc)):
911 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
912 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
916 dataset.badAmps.append(ampName)
917 dataset.gain[ampName] = np.nan
918 dataset.gainErr[ampName] = np.nan
919 dataset.noise[ampName] = np.nan
920 dataset.noiseErr[ampName] = np.nan
921 dataset.ptcFitPars[ampName] = np.nan
922 dataset.ptcFitParsError[ampName] = np.nan
923 dataset.ptcFitReducedChiSquared[ampName] = np.nan
927 if self.config.doFitBootstrap:
928 parsFit, parsFitErr, reducedChiSqPtc =
fitBootstrap(parsIniPtc, meanVecFinal,
929 varVecFinal, ptcFunc,
930 weightsY=1./np.sqrt(varVecFinal))
932 parsFit, parsFitErr, reducedChiSqPtc =
fitLeastSq(parsIniPtc, meanVecFinal,
933 varVecFinal, ptcFunc,
934 weightsY=1./np.sqrt(varVecFinal))
935 dataset.ptcFitPars[ampName] = parsFit
936 dataset.ptcFitParsError[ampName] = parsFitErr
937 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
939 if ptcFitType ==
'EXPAPPROXIMATION':
941 ptcGainErr = parsFitErr[1]
942 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
943 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
944 if ptcFitType ==
'POLYNOMIAL':
945 ptcGain = 1./parsFit[1]
946 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
947 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
948 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
949 dataset.gain[ampName] = ptcGain
950 dataset.gainErr[ampName] = ptcGainErr
951 dataset.noise[ampName] = ptcNoise
952 dataset.noiseErr[ampName] = ptcNoiseErr
953 if not len(dataset.ptcFitType) == 0:
954 dataset.ptcFitType = ptcFitType