23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
29 from sqlite3
import OperationalError
30 from collections
import Counter
31 from dataclasses
import dataclass
34 import lsst.pex.config
as pexConfig
36 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
37 checkExpLengthEqual, fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
38 from scipy.optimize
import least_squares
43 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect, fitData)
47 """Config class for photon transfer curve measurement task"""
48 ccdKey = pexConfig.Field(
50 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
53 ptcFitType = pexConfig.ChoiceField(
55 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
58 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
59 "EXPAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16).",
60 "FULLCOVARIANCE":
"Full covariances model in Astier+19 (Eq. 20)"
63 maximumRangeCovariancesAstier = pexConfig.Field(
65 doc=
"Maximum range of covariances as in Astier+19",
68 covAstierRealSpace = pexConfig.Field(
70 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
73 polynomialFitDegree = pexConfig.Field(
75 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
78 doCreateLinearizer = pexConfig.Field(
80 doc=
"Calculate non-linearity and persist linearizer?",
83 linearizerType = pexConfig.ChoiceField(
85 doc=
"Linearizer type, if doCreateLinearizer=True",
86 default=
"LINEARIZEPOLYNOMIAL",
88 "LINEARIZEPOLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegreeNonLinearity' to set 'n').",
89 "LINEARIZESQUARED":
"c0 quadratic coefficient derived from coefficients of polynomiual fit",
90 "LOOKUPTABLE":
"Loouk table formed from linear part of polynomial fit."
93 polynomialFitDegreeNonLinearity = pexConfig.Field(
95 doc=
"If doCreateLinearizer, degree of polynomial to fit the meanSignal vs exposureTime" +
96 " curve to produce the table for LinearizeLookupTable.",
99 binSize = pexConfig.Field(
101 doc=
"Bin the image by this factor in both dimensions.",
104 minMeanSignal = pexConfig.Field(
106 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
109 maxMeanSignal = pexConfig.Field(
111 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
114 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
116 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
117 " linear in the positive direction, from the PTC fit. Note that these points will also be"
118 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
119 " to allow an accurate determination of the sigmas for said iterative fit.",
124 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
126 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
127 " linear in the negative direction, from the PTC fit. Note that these points will also be"
128 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
129 " to allow an accurate determination of the sigmas for said iterative fit.",
134 sigmaCutPtcOutliers = pexConfig.Field(
136 doc=
"Sigma cut for outlier rejection in PTC.",
139 maskNameList = pexConfig.ListField(
141 doc=
"Mask list to exclude from statistics calculations.",
142 default=[
'DETECTED',
'BAD',
'NO_DATA'],
144 nSigmaClipPtc = pexConfig.Field(
146 doc=
"Sigma cut for afwMath.StatisticsControl()",
149 nIterSigmaClipPtc = pexConfig.Field(
151 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
154 maxIterationsPtcOutliers = pexConfig.Field(
156 doc=
"Maximum number of iterations for outlier rejection in PTC.",
159 doFitBootstrap = pexConfig.Field(
161 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
164 maxAduForLookupTableLinearizer = pexConfig.Field(
166 doc=
"Maximum DN value for the LookupTable linearizer.",
169 instrumentName = pexConfig.Field(
171 doc=
"Instrument name.",
178 """A simple class to hold the output from the
179 `calculateLinearityResidualAndLinearizers` function.
182 polynomialLinearizerCoefficients: list
184 quadraticPolynomialLinearizerCoefficient: float
186 linearizerTableRow: list
187 meanSignalVsTimePolyFitPars: list
188 meanSignalVsTimePolyFitParsErr: list
189 meanSignalVsTimePolyFitReducedChiSq: float
193 """A simple class to hold the output data from the PTC task.
195 The dataset is made up of a dictionary for each item, keyed by the
196 amplifiers' names, which much be supplied at construction time.
198 New items cannot be added to the class to save accidentally saving to the
199 wrong property, and the class can be frozen if desired.
201 inputVisitPairs records the visits used to produce the data.
202 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which is by definition
203 always the same length as inputVisitPairs, rawExpTimes, rawMeans
204 and rawVars, and is a list of bools, which are incrementally set to False
205 as points are discarded from the fits.
207 PTC fit parameters for polynomials are stored in a list in ascending order
208 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
209 with the length of the list corresponding to the order of the polynomial
215 List with the names of the amplifiers of the detector at hand.
218 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", or "FULLCOVARIANCE".
222 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
223 Output dataset from MeasurePhotonTransferCurveTask.
230 self.__dict__[
"ptcFitType"] = ptcFitType
231 self.__dict__[
"ampNames"] = ampNames
232 self.__dict__[
"badAmps"] = []
237 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
238 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
239 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
240 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
241 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
244 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
245 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
246 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
247 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
251 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
252 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
253 self.__dict__[
"ptcFitReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
260 self.__dict__[
"covariancesTuple"] = {ampName: []
for ampName
in ampNames}
261 self.__dict__[
"covariancesFitsWithNoB"] = {ampName: []
for ampName
in ampNames}
262 self.__dict__[
"covariancesFits"] = {ampName: []
for ampName
in ampNames}
263 self.__dict__[
"aMatrix"] = {ampName: []
for ampName
in ampNames}
264 self.__dict__[
"bMatrix"] = {ampName: []
for ampName
in ampNames}
267 self.__dict__[
"finalVars"] = {ampName: []
for ampName
in ampNames}
268 self.__dict__[
"finalModelVars"] = {ampName: []
for ampName
in ampNames}
269 self.__dict__[
"finalMeans"] = {ampName: []
for ampName
in ampNames}
272 """Protect class attributes"""
273 if attribute
not in self.__dict__:
274 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which"
275 " does not support setting of new attributes.")
277 self.__dict__[attribute] = value
280 """Get the visits used, i.e. not discarded, for a given amp.
282 If no mask has been created yet, all visits are returned.
284 if len(self.visitMask[ampName]) == 0:
285 return self.inputVisitPairs[ampName]
288 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
290 pairs = self.inputVisitPairs[ampName]
291 mask = self.visitMask[ampName]
293 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
296 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
300 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
302 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
303 used in astronomical detectors characterization (e.g., Janesick 2001,
304 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
305 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
306 times. The difference image of each pair is formed to eliminate fixed pattern noise,
307 and then the variance of the difference image and the mean of the average image
308 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
309 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
310 arXiv:1905.08677) can be fitted to the PTC curve. These models include
311 parameters such as the gain (e/DN) and readout noise.
313 Linearizers to correct for signal-chain non-linearity are also calculated.
314 The `Linearizer` class, in general, can support per-amp linearizers, but in this
315 task this is not supported.
317 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
318 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
319 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
326 Positional arguments passed to the Task constructor. None used at this
329 Keyword arguments passed on to the Task constructor. None used at this
334 RunnerClass = PairedVisitListTaskRunner
335 ConfigClass = MeasurePhotonTransferCurveTaskConfig
336 _DefaultName =
"measurePhotonTransferCurve"
339 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
340 plt.interactive(
False)
341 self.config.validate()
345 def _makeArgumentParser(cls):
346 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
348 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
349 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
350 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
351 ContainerClass=NonexistentDatasetTaskDataIdContainer,
352 help=
"The ccds to use, e.g. --id ccd=0..100")
357 """Run the Photon Transfer Curve (PTC) measurement task.
359 For a dataRef (which is each detector here),
360 and given a list of visit pairs (postISR) at different exposure times,
365 dataRef : list of lsst.daf.persistence.ButlerDataRef
366 dataRef for the detector for the visits to be fit.
368 visitPairs : `iterable` of `tuple` of `int`
369 Pairs of visit numbers to be processed together
373 detNum = dataRef.dataId[self.config.ccdKey]
374 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
380 for name
in dataRef.getButler().getKeys(
'bias'):
381 if name
not in dataRef.dataId:
383 dataRef.dataId[name] = \
384 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
385 except OperationalError:
388 amps = detector.getAmplifiers()
389 ampNames = [amp.getName()
for amp
in amps]
391 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detector.getId()))
395 for (v1, v2)
in visitPairs:
397 dataRef.dataId[
'expId'] = v1
398 exp1 = dataRef.get(
"postISRCCD", immediate=
True)
399 dataRef.dataId[
'expId'] = v2
400 exp2 = dataRef.get(
"postISRCCD", immediate=
True)
401 del dataRef.dataId[
'expId']
404 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
407 for ampNumber, amp
in enumerate(detector):
408 ampName = amp.getName()
410 doRealSpace = self.config.covAstierRealSpace
411 muDiff, varDiff, covAstier = self.
measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
412 covAstierRealSpace=doRealSpace)
414 datasetPtc.rawExpTimes[ampName].append(expTime)
415 datasetPtc.rawMeans[ampName].append(muDiff)
416 datasetPtc.rawVars[ampName].append(varDiff)
417 datasetPtc.inputVisitPairs[ampName].append((v1, v2))
419 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName)
for covRow
in covAstier]
420 tags = [
'mu',
'i',
'j',
'var',
'cov',
'npix',
'ext',
'expTime',
'ampName']
422 tupleRecords += tupleRows
423 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
425 if self.config.ptcFitType
in [
"FULLCOVARIANCE", ]:
428 elif self.config.ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
431 datasetPtc = self.
fitPtc(datasetPtc, self.config.ptcFitType)
434 if self.config.doCreateLinearizer:
435 numberAmps = len(amps)
436 numberAduValues = self.config.maxAduForLookupTableLinearizer
437 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
445 tableArray=lookupTableArray,
448 if self.config.linearizerType ==
"LINEARIZEPOLYNOMIAL":
449 linDataType =
'linearizePolynomial'
450 linMsg =
"polynomial (coefficients for a polynomial correction)."
451 elif self.config.linearizerType ==
"LINEARIZESQUARED":
452 linDataType =
'linearizePolynomial'
453 linMsg =
"squared (c0, derived from k_i coefficients of a polynomial fit)."
454 elif self.config.linearizerType ==
"LOOKUPTABLE":
455 linDataType =
'linearizePolynomial'
456 linMsg =
"lookup table (linear component of polynomial fit)."
458 raise RuntimeError(
"Invalid config.linearizerType {selg.config.linearizerType}. "
459 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
461 butler = dataRef.getButler()
462 self.log.info(f
"Writing linearizer: \n {linMsg}")
464 detName = detector.getName()
465 now = datetime.datetime.utcnow()
466 calibDate = now.strftime(
"%Y-%m-%d")
468 butler.put(linearizer, datasetType=linDataType, dataId={
'detector': detNum,
469 'detectorName': detName,
'calibDate': calibDate})
471 self.log.info(f
"Writing PTC data to {dataRef.getUri(write=True)}")
472 dataRef.put(datasetPtc, datasetType=
"photonTransferCurveDataset")
474 return pipeBase.Struct(exitStatus=0)
477 """Fit measured flat covariances to full model in Astier+19.
481 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
482 The dataset containing information such as the means, variances and exposure times.
484 covariancesWithTagsArray : `numpy.recarray`
485 Tuple with at least (mu, cov, var, i, j, npix), where:
486 mu : 0.5*(m1 + m2), where:
487 mu1: mean value of flat1
488 mu2: mean value of flat2
489 cov: covariance value at lag(i, j)
490 var: variance(covariance value at lag(0, 0))
493 npix: number of pixels used for covariance calculation.
497 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
498 This is the same dataset as the input paramter, however, it has been modified
499 to include information such as the fit vectors and the fit parameters. See
500 the class `PhotonTransferCurveDatase`.
503 covFits, covFitsNoB =
fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
504 r=self.config.maximumRangeCovariancesAstier)
506 dataset.covariancesTuple = covariancesWithTagsArray
507 dataset.covariancesFits = covFits
508 dataset.covariancesFitsWithNoB = covFitsNoB
514 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
518 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
519 The dataset containing information such as the means, variances and exposure times.
522 Dictionary of CovFit objects, with amp names as keys.
526 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
527 This is the same dataset as the input paramter, however, it has been modified
528 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
529 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
530 See the class `PhotonTransferCurveDatase`.
533 for i, amp
in enumerate(covFits):
535 meanVecFinal, varVecFinal, varVecModel, wc = fit.getNormalizedFitData(0, 0, divideByMu=
False)
537 dataset.visitMask[amp] = fit.getMaskVar()
538 dataset.gain[amp] = gain
539 dataset.gainErr[amp] = fit.getGainErr()
540 dataset.noise[amp] = np.sqrt(np.fabs(fit.getRon()))
541 dataset.noiseErr[amp] = fit.getRonErr()
542 dataset.finalVars[amp].append(varVecFinal/(gain**2))
543 dataset.finalModelVars[amp].append(varVecModel/(gain**2))
544 dataset.finalMeans[amp].append(meanVecFinal/gain)
545 dataset.aMatrix[amp].append(fit.getA())
546 dataset.bMatrix[amp].append(fit.getB())
551 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
553 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
554 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
555 keep one (covariance).
559 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
560 First exposure of flat field pair.
562 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
563 Second exposure of flat field pair.
565 region : `lsst.geom.Box2I`, optional
566 Region of each exposure where to perform the calculations (e.g, an amplifier).
568 covAstierRealSpace : `bool`, optional
569 Should the covariannces in Astier+19 be calculated in real space or via FFT?
570 See Appendix A of Astier+19.
575 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
579 Half of the clipped variance of the difference of the regions inthe two input
582 covDiffAstier : `list`
583 List with tuples of the form (dx, dy, var, cov, npix), where:
589 Variance at (dx, dy).
591 Covariance at (dx, dy).
593 Number of pixel pairs used to evaluate var and cov.
596 if region
is not None:
597 im1Area = exposure1.maskedImage[region]
598 im2Area = exposure2.maskedImage[region]
600 im1Area = exposure1.maskedImage
601 im2Area = exposure2.maskedImage
603 im1Area = afwMath.binImage(im1Area, self.config.binSize)
604 im2Area = afwMath.binImage(im2Area, self.config.binSize)
606 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
607 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
608 self.config.nIterSigmaClipPtc,
610 im1StatsCtrl.setAndMask(im1MaskVal)
612 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
613 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
614 self.config.nIterSigmaClipPtc,
616 im2StatsCtrl.setAndMask(im2MaskVal)
619 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
620 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
625 temp = im2Area.clone()
627 diffIm = im1Area.clone()
632 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
633 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
634 self.config.nIterSigmaClipPtc,
636 diffImStatsCtrl.setAndMask(diffImMaskVal)
638 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
641 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
642 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
645 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
648 maxRangeCov = self.config.maximumRangeCovariancesAstier
649 if covAstierRealSpace:
650 covDiffAstier =
computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
652 shapeDiff = diffIm.getImage().getArray().shape
653 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
654 c =
CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
655 covDiffAstier = c.reportCovFft(maxRangeCov)
657 return mu, varDiff, covDiffAstier
660 """Compute covariances of diffImage in real space.
662 For lags larger than ~25, it is slower than the FFT way.
663 Taken from https://github.com/PierreAstier/bfptc/
667 diffImage : `numpy.array`
668 Image to compute the covariance of.
670 weightImage : `numpy.array`
671 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
674 Last index of the covariance to be computed.
679 List with tuples of the form (dx, dy, var, cov, npix), where:
685 Variance at (dx, dy).
687 Covariance at (dx, dy).
689 Number of pixel pairs used to evaluate var and cov.
694 for dy
in range(maxRange + 1):
695 for dx
in range(0, maxRange + 1):
698 cov2, nPix2 = self.
covDirectValue(diffImage, weightImage, dx, -dy)
699 cov = 0.5*(cov1 + cov2)
703 if (dx == 0
and dy == 0):
705 outList.append((dx, dy, var, cov, nPix))
710 """Compute covariances of diffImage in real space at lag (dx, dy).
712 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
716 diffImage : `numpy.array`
717 Image to compute the covariance of.
719 weightImage : `numpy.array`
720 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
731 Covariance at (dx, dy)
734 Number of pixel pairs used to evaluate var and cov.
736 (nCols, nRows) = diffImage.shape
740 (dx, dy) = (-dx, -dy)
744 im1 = diffImage[dy:, dx:]
745 w1 = weightImage[dy:, dx:]
746 im2 = diffImage[:nCols - dy, :nRows - dx]
747 w2 = weightImage[:nCols - dy, :nRows - dx]
749 im1 = diffImage[:nCols + dy, dx:]
750 w1 = weightImage[:nCols + dy, dx:]
751 im2 = diffImage[-dy:, :nRows - dx]
752 w2 = weightImage[-dy:, :nRows - dx]
758 s1 = im1TimesW.sum()/nPix
759 s2 = (im2*wAll).sum()/nPix
760 p = (im1TimesW*im2).sum()/nPix
766 """Fit non-linearity function and build linearizer objects.
770 datasePtct : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
771 The dataset containing information such as the means, variances and exposure times.
774 detector : `lsst.afw.cameraGeom.Detector`
777 tableArray : `np.array`, optional
778 Optional. Look-up table array with size rows=nAmps and columns=DN values.
779 It will be modified in-place if supplied.
781 log : `lsst.log.Log`, optional
782 Logger to handle messages.
786 linearizer : `lsst.ip.isr.Linearizer`
791 datasetNonLinearity = self.
fitNonLinearity(datasetPtc, tableArray=tableArray)
794 now = datetime.datetime.utcnow()
795 calibDate = now.strftime(
"%Y-%m-%d")
796 linType = self.config.linearizerType
798 if linType ==
"LOOKUPTABLE":
799 tableArray = tableArray
804 instruName=self.config.instrumentName,
805 tableArray=tableArray,
811 """Fit a polynomial to signal vs effective time curve to calculate linearity and residuals.
815 datasetPtc : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
816 The dataset containing the means, variances and exposure times.
818 tableArray : `np.array`
819 Optional. Look-up table array with size rows=nAmps and columns=DN values.
820 It will be modified in-place if supplied.
824 datasetNonLinearity : `dict`
825 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
826 dataclasses. Each one holds the output of `calculateLinearityResidualAndLinearizers` per
829 datasetNonLinearity = {ampName: []
for ampName
in datasetPtc.ampNames}
830 for i, ampName
in enumerate(datasetPtc.ampNames):
832 if (len(datasetPtc.visitMask[ampName]) == 0):
833 self.log.warn(f
"Mask not found for {ampName} in non-linearity fit. Using all points.")
834 mask = np.repeat(
True, len(datasetPtc.rawExpTimes[ampName]))
836 mask = datasetPtc.visitMask[ampName]
838 timeVecFinal = np.array(datasetPtc.rawExpTimes[ampName])[mask]
839 meanVecFinal = np.array(datasetPtc.rawMeans[ampName])[mask]
846 if tableArray
is not None:
847 tableArray[i, :] = datasetLinRes.linearizerTableRow
849 datasetNonLinearity[ampName] = datasetLinRes
851 return datasetNonLinearity
854 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
855 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
856 to populate LinearizeLookupTable.
857 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
858 and LinearizeSquared."
863 exposureTimeVector: `list` of `float`
864 List of exposure times for each flat pair
866 meanSignalVector: `list` of `float`
867 List of mean signal from diference image of flat pairs
871 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
872 The dataset containing the fit parameters, the NL correction coefficients, and the
873 LUT row for the amplifier at hand.
879 dataset.polynomialLinearizerCoefficients : `list` of `float`
880 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
882 c_(j-2) = -k_j/(k_1^j) with units DN^(1-j) (c.f., Eq. 37 of 2003.05978). The units of k_j are
883 DN/t^j, and they are fit from meanSignalVector = k0 + k1*exposureTimeVector +
884 k2*exposureTimeVector^2 + ... + kn*exposureTimeVector^n, with
885 n = "polynomialFitDegreeNonLinearity". k_0 and k_1 and degenerate with bias level and gain,
886 and are not used by the non-linearity correction. Therefore, j = 2...n in the above expression
887 (see `LinearizePolynomial` class in `linearize.py`.)
889 dataset.quadraticPolynomialLinearizerCoefficient : `float`
890 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
891 c0 = -k2/(k1^2), where k1 and k2 are fit from
892 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
893 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
895 dataset.linearizerTableRow : `list` of `float`
896 One dimensional array with deviation from linear part of n-order polynomial fit
897 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
898 of the table array for LinearizeLookupTable.
900 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
901 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
903 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
904 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
906 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
907 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
912 if self.config.doFitBootstrap:
913 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit =
fitBootstrap(parsIniNonLinearity,
918 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit =
fitLeastSq(parsIniNonLinearity,
925 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
926 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
927 signalIdeal = parsFit[0] + parsFit[1]*timeRange
929 linearizerTableRow = signalIdeal - signalUncorrected
935 polynomialLinearizerCoefficients = []
936 for i, coefficient
in enumerate(parsFit):
937 c = -coefficient/(k1**i)
938 polynomialLinearizerCoefficients.append(c)
939 if np.fabs(c) > 1e-10:
940 msg = f
"Coefficient {c} in polynomial fit larger than threshold 1e-10."
943 c0 = polynomialLinearizerCoefficients[2]
946 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
947 dataset.quadraticPolynomialLinearizerCoefficient = c0
948 dataset.linearizerTableRow = linearizerTableRow
949 dataset.meanSignalVsTimePolyFitPars = parsFit
950 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
951 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
956 tableArray=None, log=None):
957 """Build linearizer object to persist.
961 datasetNonLinearity : `dict`
962 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` objects.
964 detector : `lsst.afw.cameraGeom.Detector`
967 calibDate : `datetime.datetime`
970 linearizerType : `str`
971 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
973 instruName : `str`, optional
976 tableArray : `np.array`, optional
977 Look-up table array with size rows=nAmps and columns=DN values
979 log : `lsst.log.Log`, optional
980 Logger to handle messages
984 linearizer : `lsst.ip.isr.Linearizer`
987 detName = detector.getName()
988 detNum = detector.getId()
989 if linearizerType ==
"LOOKUPTABLE":
990 if tableArray
is not None:
991 linearizer =
Linearizer(detector=detector, table=tableArray, log=log)
993 raise RuntimeError(
"tableArray must be provided when creating a LookupTable linearizer")
994 elif linearizerType
in (
"LINEARIZESQUARED",
"LINEARIZEPOLYNOMIAL"):
997 raise RuntimeError(
"Invalid linearizerType {linearizerType} to build a Linearizer object. "
998 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
999 for i, amp
in enumerate(detector.getAmplifiers()):
1000 ampName = amp.getName()
1001 datasetNonLinAmp = datasetNonLinearity[ampName]
1002 if linearizerType ==
"LOOKUPTABLE":
1003 linearizer.linearityCoeffs[ampName] = [i, 0]
1004 linearizer.linearityType[ampName] =
"LookupTable"
1005 elif linearizerType ==
"LINEARIZESQUARED":
1006 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1007 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1008 linearizer.linearityFitReducedChiSquared[ampName] = (
1009 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1010 linearizer.linearityCoeffs[ampName] = [
1011 datasetNonLinAmp.quadraticPolynomialLinearizerCoefficient]
1012 linearizer.linearityType[ampName] =
"Squared"
1013 elif linearizerType ==
"LINEARIZEPOLYNOMIAL":
1014 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1015 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1016 linearizer.linearityFitReducedChiSquared[ampName] = (
1017 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1021 polyLinCoeffs = np.array(datasetNonLinAmp.polynomialLinearizerCoefficients[2:])
1022 linearizer.linearityCoeffs[ampName] = polyLinCoeffs
1023 linearizer.linearityType[ampName] =
"Polynomial"
1024 linearizer.linearityBBox[ampName] = amp.getBBox()
1025 linearizer.validate()
1026 calibId = f
"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
1029 raftName = detName.split(
"_")[0]
1030 calibId += f
" raftName={raftName}"
1033 calibId += f
" raftName={raftname}"
1035 serial = detector.getSerial()
1036 linearizer.updateMetadata(instrumentName=instruName, detectorId=f
"{detNum}",
1037 calibId=calibId, serial=serial, detectorName=f
"{detName}")
1042 def _initialParsForPolynomial(order):
1044 pars = np.zeros(order, dtype=np.float)
1051 def _boundsForPolynomial(initialPars):
1052 lowers = [np.NINF
for p
in initialPars]
1053 uppers = [np.inf
for p
in initialPars]
1055 return (lowers, uppers)
1058 def _boundsForAstier(initialPars):
1059 lowers = [np.NINF
for p
in initialPars]
1060 uppers = [np.inf
for p
in initialPars]
1061 return (lowers, uppers)
1064 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
1065 """Return a boolean array to mask bad points.
1067 A linear function has a constant ratio, so find the median
1068 value of the ratios, and exclude the points that deviate
1069 from that by more than a factor of maxDeviationPositive/negative.
1070 Asymmetric deviations are supported as we expect the PTC to turn
1071 down as the flux increases, but sometimes it anomalously turns
1072 upwards just before turning over, which ruins the fits, so it
1073 is wise to be stricter about restricting positive outliers than
1076 Too high and points that are so bad that fit will fail will be included
1077 Too low and the non-linear points will be excluded, biasing the NL fit."""
1078 ratios = [b/a
for (a, b)
in zip(means, variances)]
1079 medianRatio = np.median(ratios)
1080 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
1083 maxDeviationPositive = abs(maxDeviationPositive)
1084 maxDeviationNegative = -1. * abs(maxDeviationNegative)
1086 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
1087 else False for r
in ratioDeviations])
1090 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
1092 nBad = Counter(array)[0]
1097 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
1100 array[array == 0] = substituteValue
1104 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
1106 Fit the photon transfer curve with either a polynomial of the order
1107 specified in the task config, or using the Astier approximation.
1109 Sigma clipping is performed iteratively for the fit, as well as an
1110 initial clipping of data points that are more than
1111 config.initialNonLinearityExclusionThreshold away from lying on a
1112 straight line. This other step is necessary because the photon transfer
1113 curve turns over catastrophically at very high flux (because saturation
1114 drops the variance to ~0) and these far outliers cause the initial fit
1115 to fail, meaning the sigma cannot be calculated to perform the
1120 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1121 The dataset containing the means, variances and exposure times
1124 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1125 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
1129 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1130 This is the same dataset as the input paramter, however, it has been modified
1131 to include information such as the fit vectors and the fit parameters. See
1132 the class `PhotonTransferCurveDatase`.
1135 def errFunc(p, x, y):
1136 return ptcFunc(p, x) - y
1138 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1139 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1141 for i, ampName
in enumerate(dataset.ampNames):
1142 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
1143 meanVecOriginal = np.array(dataset.rawMeans[ampName])
1144 varVecOriginal = np.array(dataset.rawVars[ampName])
1147 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
1148 (meanVecOriginal <= self.config.maxMeanSignal))
1151 self.config.initialNonLinearityExclusionThresholdPositive,
1152 self.config.initialNonLinearityExclusionThresholdNegative)
1153 mask = mask & goodPoints
1155 if ptcFitType ==
'EXPAPPROXIMATION':
1156 ptcFunc = funcAstier
1157 parsIniPtc = [-1e-9, 1.0, 10.]
1159 if ptcFitType ==
'POLYNOMIAL':
1160 ptcFunc = funcPolynomial
1166 while count <= maxIterationsPtcOutliers:
1170 meanTempVec = meanVecOriginal[mask]
1171 varTempVec = varVecOriginal[mask]
1172 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1178 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1179 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
1180 mask = mask & newMask
1182 nDroppedTotal = Counter(mask)[
False]
1183 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1186 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1188 dataset.visitMask[ampName] = mask
1190 meanVecFinal = meanVecOriginal[mask]
1191 varVecFinal = varVecOriginal[mask]
1193 if Counter(mask)[
False] > 0:
1194 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
1195 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1197 if (len(meanVecFinal) < len(parsIniPtc)):
1198 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1199 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1203 dataset.badAmps.append(ampName)
1204 dataset.gain[ampName] = np.nan
1205 dataset.gainErr[ampName] = np.nan
1206 dataset.noise[ampName] = np.nan
1207 dataset.noiseErr[ampName] = np.nan
1208 dataset.ptcFitPars[ampName] = np.nan
1209 dataset.ptcFitParsError[ampName] = np.nan
1210 dataset.ptcFitReducedChiSquared[ampName] = np.nan
1214 if self.config.doFitBootstrap:
1215 parsFit, parsFitErr, reducedChiSqPtc =
fitBootstrap(parsIniPtc, meanVecFinal,
1216 varVecFinal, ptcFunc)
1218 parsFit, parsFitErr, reducedChiSqPtc =
fitLeastSq(parsIniPtc, meanVecFinal,
1219 varVecFinal, ptcFunc)
1220 dataset.ptcFitPars[ampName] = parsFit
1221 dataset.ptcFitParsError[ampName] = parsFitErr
1222 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1224 if ptcFitType ==
'EXPAPPROXIMATION':
1225 ptcGain = parsFit[1]
1226 ptcGainErr = parsFitErr[1]
1227 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1228 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1229 if ptcFitType ==
'POLYNOMIAL':
1230 ptcGain = 1./parsFit[1]
1231 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1232 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1233 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1234 dataset.gain[ampName] = ptcGain
1235 dataset.gainErr[ampName] = ptcGainErr
1236 dataset.noise[ampName] = ptcNoise
1237 dataset.noiseErr[ampName] = ptcNoiseErr
1238 if not len(dataset.ptcFitType) == 0:
1239 dataset.ptcFitType = ptcFitType