23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
29 from sqlite3
import OperationalError
30 from collections
import Counter
31 from dataclasses
import dataclass
34 import lsst.pex.config
as pexConfig
36 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
37 checkExpLengthEqual, fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
38 from scipy.optimize
import least_squares
43 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect, fitData)
47 """Config class for photon transfer curve measurement task"""
48 ccdKey = pexConfig.Field(
50 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
53 ptcFitType = pexConfig.ChoiceField(
55 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
58 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
59 "EXPAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16).",
60 "FULLCOVARIANCE":
"Full covariances model in Astier+19 (Eq. 20)"
63 maximumRangeCovariancesAstier = pexConfig.Field(
65 doc=
"Maximum range of covariances as in Astier+19",
68 covAstierRealSpace = pexConfig.Field(
70 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
73 polynomialFitDegree = pexConfig.Field(
75 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
78 doCreateLinearizer = pexConfig.Field(
80 doc=
"Calculate non-linearity and persist linearizer?",
83 linearizerType = pexConfig.ChoiceField(
85 doc=
"Linearizer type, if doCreateLinearizer=True",
86 default=
"LINEARIZEPOLYNOMIAL",
88 "LINEARIZEPOLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegreeNonLinearity' to set 'n').",
89 "LINEARIZESQUARED":
"c0 quadratic coefficient derived from coefficients of polynomiual fit",
90 "LOOKUPTABLE":
"Loouk table formed from linear part of polynomial fit."
93 polynomialFitDegreeNonLinearity = pexConfig.Field(
95 doc=
"If doCreateLinearizer, degree of polynomial to fit the meanSignal vs exposureTime" +
96 " curve to produce the table for LinearizeLookupTable.",
99 binSize = pexConfig.Field(
101 doc=
"Bin the image by this factor in both dimensions.",
104 minMeanSignal = pexConfig.Field(
106 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
109 maxMeanSignal = pexConfig.Field(
111 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
114 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
116 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
117 " linear in the positive direction, from the PTC fit. Note that these points will also be"
118 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
119 " to allow an accurate determination of the sigmas for said iterative fit.",
124 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
126 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
127 " linear in the negative direction, from the PTC fit. Note that these points will also be"
128 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
129 " to allow an accurate determination of the sigmas for said iterative fit.",
134 sigmaCutPtcOutliers = pexConfig.Field(
136 doc=
"Sigma cut for outlier rejection in PTC.",
139 nSigmaClipPtc = pexConfig.Field(
141 doc=
"Sigma cut for afwMath.StatisticsControl()",
144 nIterSigmaClipPtc = pexConfig.Field(
146 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
149 maxIterationsPtcOutliers = pexConfig.Field(
151 doc=
"Maximum number of iterations for outlier rejection in PTC.",
154 doFitBootstrap = pexConfig.Field(
156 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
159 maxAduForLookupTableLinearizer = pexConfig.Field(
161 doc=
"Maximum DN value for the LookupTable linearizer.",
164 instrumentName = pexConfig.Field(
166 doc=
"Instrument name.",
173 """A simple class to hold the output from the
174 `calculateLinearityResidualAndLinearizers` function.
177 polynomialLinearizerCoefficients: list
179 quadraticPolynomialLinearizerCoefficient: float
181 linearizerTableRow: list
182 meanSignalVsTimePolyFitPars: list
183 meanSignalVsTimePolyFitParsErr: list
184 meanSignalVsTimePolyFitReducedChiSq: float
188 """A simple class to hold the output data from the PTC task.
190 The dataset is made up of a dictionary for each item, keyed by the
191 amplifiers' names, which much be supplied at construction time.
193 New items cannot be added to the class to save accidentally saving to the
194 wrong property, and the class can be frozen if desired.
196 inputVisitPairs records the visits used to produce the data.
197 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which is by definition
198 always the same length as inputVisitPairs, rawExpTimes, rawMeans
199 and rawVars, and is a list of bools, which are incrementally set to False
200 as points are discarded from the fits.
202 PTC fit parameters for polynomials are stored in a list in ascending order
203 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
204 with the length of the list corresponding to the order of the polynomial
210 List with the names of the amplifiers of the detector at hand.
213 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", or "FULLCOVARIANCE".
217 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
218 Output dataset from MeasurePhotonTransferCurveTask.
225 self.__dict__[
"ptcFitType"] = ptcFitType
226 self.__dict__[
"ampNames"] = ampNames
227 self.__dict__[
"badAmps"] = []
232 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
233 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
234 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
235 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
236 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
239 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
240 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
241 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
242 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
246 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
247 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
248 self.__dict__[
"ptcFitReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
255 self.__dict__[
"covariancesTuple"] = {ampName: []
for ampName
in ampNames}
256 self.__dict__[
"covariancesFitsWithNoB"] = {ampName: []
for ampName
in ampNames}
257 self.__dict__[
"covariancesFits"] = {ampName: []
for ampName
in ampNames}
258 self.__dict__[
"aMatrix"] = {ampName: []
for ampName
in ampNames}
259 self.__dict__[
"bMatrix"] = {ampName: []
for ampName
in ampNames}
262 self.__dict__[
"finalVars"] = {ampName: []
for ampName
in ampNames}
263 self.__dict__[
"finalModelVars"] = {ampName: []
for ampName
in ampNames}
264 self.__dict__[
"finalMeans"] = {ampName: []
for ampName
in ampNames}
267 """Protect class attributes"""
268 if attribute
not in self.__dict__:
269 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which"
270 " does not support setting of new attributes.")
272 self.__dict__[attribute] = value
275 """Get the visits used, i.e. not discarded, for a given amp.
277 If no mask has been created yet, all visits are returned.
279 if len(self.visitMask[ampName]) == 0:
280 return self.inputVisitPairs[ampName]
283 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
285 pairs = self.inputVisitPairs[ampName]
286 mask = self.visitMask[ampName]
288 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
291 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
295 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
297 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
298 used in astronomical detectors characterization (e.g., Janesick 2001,
299 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
300 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
301 times. The difference image of each pair is formed to eliminate fixed pattern noise,
302 and then the variance of the difference image and the mean of the average image
303 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
304 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
305 arXiv:1905.08677) can be fitted to the PTC curve. These models include
306 parameters such as the gain (e/DN) and readout noise.
308 Linearizers to correct for signal-chain non-linearity are also calculated.
309 The `Linearizer` class, in general, can support per-amp linearizers, but in this
310 task this is not supported.
312 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
313 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
314 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
321 Positional arguments passed to the Task constructor. None used at this
324 Keyword arguments passed on to the Task constructor. None used at this
329 RunnerClass = PairedVisitListTaskRunner
330 ConfigClass = MeasurePhotonTransferCurveTaskConfig
331 _DefaultName =
"measurePhotonTransferCurve"
334 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
335 plt.interactive(
False)
336 self.config.validate()
340 def _makeArgumentParser(cls):
341 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
343 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
344 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
345 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
346 ContainerClass=NonexistentDatasetTaskDataIdContainer,
347 help=
"The ccds to use, e.g. --id ccd=0..100")
352 """Run the Photon Transfer Curve (PTC) measurement task.
354 For a dataRef (which is each detector here),
355 and given a list of visit pairs (postISR) at different exposure times,
360 dataRef : list of lsst.daf.persistence.ButlerDataRef
361 dataRef for the detector for the visits to be fit.
363 visitPairs : `iterable` of `tuple` of `int`
364 Pairs of visit numbers to be processed together
368 detNum = dataRef.dataId[self.config.ccdKey]
369 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
375 for name
in dataRef.getButler().getKeys(
'bias'):
376 if name
not in dataRef.dataId:
378 dataRef.dataId[name] = \
379 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
380 except OperationalError:
383 amps = detector.getAmplifiers()
384 ampNames = [amp.getName()
for amp
in amps]
386 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detector.getId()))
390 for (v1, v2)
in visitPairs:
392 dataRef.dataId[
'expId'] = v1
393 exp1 = dataRef.get(
"postISRCCD", immediate=
True)
394 dataRef.dataId[
'expId'] = v2
395 exp2 = dataRef.get(
"postISRCCD", immediate=
True)
396 del dataRef.dataId[
'expId']
399 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
402 for ampNumber, amp
in enumerate(detector):
403 ampName = amp.getName()
405 doRealSpace = self.config.covAstierRealSpace
406 muDiff, varDiff, covAstier = self.
measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
407 covAstierRealSpace=doRealSpace)
409 datasetPtc.rawExpTimes[ampName].append(expTime)
410 datasetPtc.rawMeans[ampName].append(muDiff)
411 datasetPtc.rawVars[ampName].append(varDiff)
412 datasetPtc.inputVisitPairs[ampName].append((v1, v2))
414 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName)
for covRow
in covAstier]
415 tags = [
'mu',
'i',
'j',
'var',
'cov',
'npix',
'ext',
'expTime',
'ampName']
417 tupleRecords += tupleRows
418 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
420 if self.config.ptcFitType
in [
"FULLCOVARIANCE", ]:
423 elif self.config.ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
426 datasetPtc = self.
fitPtc(datasetPtc, self.config.ptcFitType)
429 if self.config.doCreateLinearizer:
430 numberAmps = len(amps)
431 numberAduValues = self.config.maxAduForLookupTableLinearizer
432 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
440 tableArray=lookupTableArray,
443 if self.config.linearizerType ==
"LINEARIZEPOLYNOMIAL":
444 linDataType =
'linearizePolynomial'
445 linMsg =
"polynomial (coefficients for a polynomial correction)."
446 elif self.config.linearizerType ==
"LINEARIZESQUARED":
447 linDataType =
'linearizePolynomial'
448 linMsg =
"squared (c0, derived from k_i coefficients of a polynomial fit)."
449 elif self.config.linearizerType ==
"LOOKUPTABLE":
450 linDataType =
'linearizePolynomial'
451 linMsg =
"lookup table (linear component of polynomial fit)."
453 raise RuntimeError(
"Invalid config.linearizerType {selg.config.linearizerType}. "
454 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
456 butler = dataRef.getButler()
457 self.log.info(f
"Writing linearizer: \n {linMsg}")
459 detName = detector.getName()
460 now = datetime.datetime.utcnow()
461 calibDate = now.strftime(
"%Y-%m-%d")
463 butler.put(linearizer, datasetType=linDataType, dataId={
'detector': detNum,
464 'detectorName': detName,
'calibDate': calibDate})
466 self.log.info(f
"Writing PTC data to {dataRef.getUri(write=True)}")
467 dataRef.put(datasetPtc, datasetType=
"photonTransferCurveDataset")
469 return pipeBase.Struct(exitStatus=0)
472 """Fit measured flat covariances to full model in Astier+19.
476 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
477 The dataset containing information such as the means, variances and exposure times.
479 covariancesWithTagsArray : `numpy.recarray`
480 Tuple with at least (mu, cov, var, i, j, npix), where:
481 mu : 0.5*(m1 + m2), where:
482 mu1: mean value of flat1
483 mu2: mean value of flat2
484 cov: covariance value at lag(i, j)
485 var: variance(covariance value at lag(0, 0))
488 npix: number of pixels used for covariance calculation.
492 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
493 This is the same dataset as the input paramter, however, it has been modified
494 to include information such as the fit vectors and the fit parameters. See
495 the class `PhotonTransferCurveDatase`.
498 covFits, covFitsNoB =
fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
499 r=self.config.maximumRangeCovariancesAstier)
501 dataset.covariancesTuple = covariancesWithTagsArray
502 dataset.covariancesFits = covFits
503 dataset.covariancesFitsWithNoB = covFitsNoB
509 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
513 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
514 The dataset containing information such as the means, variances and exposure times.
517 Dictionary of CovFit objects, with amp names as keys.
521 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
522 This is the same dataset as the input paramter, however, it has been modified
523 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
524 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
525 See the class `PhotonTransferCurveDatase`.
528 for i, amp
in enumerate(covFits):
530 meanVecFinal, varVecFinal, varVecModel, wc = fit.getNormalizedFitData(0, 0, divideByMu=
False)
532 dataset.visitMask[amp] = fit.getMaskVar()
533 dataset.gain[amp] = gain
534 dataset.gainErr[amp] = fit.getGainErr()
535 dataset.noise[amp] = np.sqrt(np.fabs(fit.getRon()))
536 dataset.noiseErr[amp] = fit.getRonErr()
537 dataset.finalVars[amp].append(varVecFinal/(gain**2))
538 dataset.finalModelVars[amp].append(varVecModel/(gain**2))
539 dataset.finalMeans[amp].append(meanVecFinal/gain)
540 dataset.aMatrix[amp].append(fit.getA())
541 dataset.bMatrix[amp].append(fit.getB())
546 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
548 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
549 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
550 keep one (covariance).
554 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
555 First exposure of flat field pair.
557 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
558 Second exposure of flat field pair.
560 region : `lsst.geom.Box2I`, optional
561 Region of each exposure where to perform the calculations (e.g, an amplifier).
563 covAstierRealSpace : `bool`, optional
564 Should the covariannces in Astier+19 be calculated in real space or via FFT?
565 See Appendix A of Astier+19.
570 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
574 Half of the clipped variance of the difference of the regions inthe two input
577 covDiffAstier : `list`
578 List with tuples of the form (dx, dy, var, cov, npix), where:
584 Variance at (dx, dy).
586 Covariance at (dx, dy).
588 Number of pixel pairs used to evaluate var and cov.
591 if region
is not None:
592 im1Area = exposure1.maskedImage[region]
593 im2Area = exposure2.maskedImage[region]
595 im1Area = exposure1.maskedImage
596 im2Area = exposure2.maskedImage
598 im1Area = afwMath.binImage(im1Area, self.config.binSize)
599 im2Area = afwMath.binImage(im2Area, self.config.binSize)
601 statsCtrl = afwMath.StatisticsControl()
602 statsCtrl.setNumSigmaClip(self.config.nSigmaClipPtc)
603 statsCtrl.setNumIter(self.config.nIterSigmaClipPtc)
605 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, statsCtrl).getValue()
606 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, statsCtrl).getValue()
611 temp = im2Area.clone()
613 diffIm = im1Area.clone()
618 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, statsCtrl).getValue())
621 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
622 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
625 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
628 maxRangeCov = self.config.maximumRangeCovariancesAstier
629 if covAstierRealSpace:
630 covDiffAstier =
computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
632 shapeDiff = diffIm.getImage().getArray().shape
633 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
634 c =
CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
635 covDiffAstier = c.reportCovFft(maxRangeCov)
637 return mu, varDiff, covDiffAstier
640 """Compute covariances of diffImage in real space.
642 For lags larger than ~25, it is slower than the FFT way.
643 Taken from https://github.com/PierreAstier/bfptc/
647 diffImage : `numpy.array`
648 Image to compute the covariance of.
650 weightImage : `numpy.array`
651 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
654 Last index of the covariance to be computed.
659 List with tuples of the form (dx, dy, var, cov, npix), where:
665 Variance at (dx, dy).
667 Covariance at (dx, dy).
669 Number of pixel pairs used to evaluate var and cov.
674 for dy
in range(maxRange + 1):
675 for dx
in range(0, maxRange + 1):
678 cov2, nPix2 = self.
covDirectValue(diffImage, weightImage, dx, -dy)
679 cov = 0.5*(cov1 + cov2)
683 if (dx == 0
and dy == 0):
685 outList.append((dx, dy, var, cov, nPix))
690 """Compute covariances of diffImage in real space at lag (dx, dy).
692 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
696 diffImage : `numpy.array`
697 Image to compute the covariance of.
699 weightImage : `numpy.array`
700 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
711 Covariance at (dx, dy)
714 Number of pixel pairs used to evaluate var and cov.
716 (nCols, nRows) = diffImage.shape
720 (dx, dy) = (-dx, -dy)
724 im1 = diffImage[dy:, dx:]
725 w1 = weightImage[dy:, dx:]
726 im2 = diffImage[:nCols - dy, :nRows - dx]
727 w2 = weightImage[:nCols - dy, :nRows - dx]
729 im1 = diffImage[:nCols + dy, dx:]
730 w1 = weightImage[:nCols + dy, dx:]
731 im2 = diffImage[-dy:, :nRows - dx]
732 w2 = weightImage[-dy:, :nRows - dx]
738 s1 = im1TimesW.sum()/nPix
739 s2 = (im2*wAll).sum()/nPix
740 p = (im1TimesW*im2).sum()/nPix
746 """Fit non-linearity function and build linearizer objects.
750 datasePtct : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
751 The dataset containing information such as the means, variances and exposure times.
754 detector : `lsst.afw.cameraGeom.Detector`
757 tableArray : `np.array`, optional
758 Optional. Look-up table array with size rows=nAmps and columns=DN values.
759 It will be modified in-place if supplied.
761 log : `lsst.log.Log`, optional
762 Logger to handle messages.
766 linearizer : `lsst.ip.isr.Linearizer`
771 datasetNonLinearity = self.
fitNonLinearity(datasetPtc, tableArray=tableArray)
774 now = datetime.datetime.utcnow()
775 calibDate = now.strftime(
"%Y-%m-%d")
776 linType = self.config.linearizerType
778 if linType ==
"LOOKUPTABLE":
779 tableArray = tableArray
784 instruName=self.config.instrumentName,
785 tableArray=tableArray,
791 """Fit a polynomial to signal vs effective time curve to calculate linearity and residuals.
795 datasetPtc : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
796 The dataset containing the means, variances and exposure times.
798 tableArray : `np.array`
799 Optional. Look-up table array with size rows=nAmps and columns=DN values.
800 It will be modified in-place if supplied.
804 datasetNonLinearity : `dict`
805 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
806 dataclasses. Each one holds the output of `calculateLinearityResidualAndLinearizers` per
809 datasetNonLinearity = {ampName: []
for ampName
in datasetPtc.ampNames}
810 for i, ampName
in enumerate(datasetPtc.ampNames):
812 if (len(datasetPtc.visitMask[ampName]) == 0):
813 self.log.warn(f
"Mask not found for {ampName} in non-linearity fit. Using all points.")
814 mask = np.repeat(
True, len(datasetPtc.rawExpTimes[ampName]))
816 mask = datasetPtc.visitMask[ampName]
818 timeVecFinal = np.array(datasetPtc.rawExpTimes[ampName])[mask]
819 meanVecFinal = np.array(datasetPtc.rawMeans[ampName])[mask]
826 if tableArray
is not None:
827 tableArray[i, :] = datasetLinRes.linearizerTableRow
829 datasetNonLinearity[ampName] = datasetLinRes
831 return datasetNonLinearity
834 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
835 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
836 to populate LinearizeLookupTable.
837 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
838 and LinearizeSquared."
843 exposureTimeVector: `list` of `float`
844 List of exposure times for each flat pair
846 meanSignalVector: `list` of `float`
847 List of mean signal from diference image of flat pairs
851 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
852 The dataset containing the fit parameters, the NL correction coefficients, and the
853 LUT row for the amplifier at hand.
859 dataset.polynomialLinearizerCoefficients : `list` of `float`
860 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
862 c_(j-2) = -k_j/(k_1^j) with units DN^(1-j) (c.f., Eq. 37 of 2003.05978). The units of k_j are
863 DN/t^j, and they are fit from meanSignalVector = k0 + k1*exposureTimeVector +
864 k2*exposureTimeVector^2 + ... + kn*exposureTimeVector^n, with
865 n = "polynomialFitDegreeNonLinearity". k_0 and k_1 and degenerate with bias level and gain,
866 and are not used by the non-linearity correction. Therefore, j = 2...n in the above expression
867 (see `LinearizePolynomial` class in `linearize.py`.)
869 dataset.quadraticPolynomialLinearizerCoefficient : `float`
870 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
871 c0 = -k2/(k1^2), where k1 and k2 are fit from
872 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
873 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
875 dataset.linearizerTableRow : `list` of `float`
876 One dimensional array with deviation from linear part of n-order polynomial fit
877 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
878 of the table array for LinearizeLookupTable.
880 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
881 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
883 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
884 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
886 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
887 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
892 if self.config.doFitBootstrap:
893 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit =
fitBootstrap(parsIniNonLinearity,
898 parsFit, parsFitErr, reducedChiSquaredNonLinearityFit =
fitLeastSq(parsIniNonLinearity,
905 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
906 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
907 signalIdeal = parsFit[0] + parsFit[1]*timeRange
909 linearizerTableRow = signalIdeal - signalUncorrected
915 polynomialLinearizerCoefficients = []
916 for i, coefficient
in enumerate(parsFit):
917 c = -coefficient/(k1**i)
918 polynomialLinearizerCoefficients.append(c)
919 if np.fabs(c) > 1e-10:
920 msg = f
"Coefficient {c} in polynomial fit larger than threshold 1e-10."
923 c0 = polynomialLinearizerCoefficients[2]
926 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
927 dataset.quadraticPolynomialLinearizerCoefficient = c0
928 dataset.linearizerTableRow = linearizerTableRow
929 dataset.meanSignalVsTimePolyFitPars = parsFit
930 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
931 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
936 tableArray=None, log=None):
937 """Build linearizer object to persist.
941 datasetNonLinearity : `dict`
942 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` objects.
944 detector : `lsst.afw.cameraGeom.Detector`
947 calibDate : `datetime.datetime`
950 linearizerType : `str`
951 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
953 instruName : `str`, optional
956 tableArray : `np.array`, optional
957 Look-up table array with size rows=nAmps and columns=DN values
959 log : `lsst.log.Log`, optional
960 Logger to handle messages
964 linearizer : `lsst.ip.isr.Linearizer`
967 detName = detector.getName()
968 detNum = detector.getId()
969 if linearizerType ==
"LOOKUPTABLE":
970 if tableArray
is not None:
971 linearizer =
Linearizer(detector=detector, table=tableArray, log=log)
973 raise RuntimeError(
"tableArray must be provided when creating a LookupTable linearizer")
974 elif linearizerType
in (
"LINEARIZESQUARED",
"LINEARIZEPOLYNOMIAL"):
977 raise RuntimeError(
"Invalid linearizerType {linearizerType} to build a Linearizer object. "
978 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
979 for i, amp
in enumerate(detector.getAmplifiers()):
980 ampName = amp.getName()
981 datasetNonLinAmp = datasetNonLinearity[ampName]
982 if linearizerType ==
"LOOKUPTABLE":
983 linearizer.linearityCoeffs[ampName] = [i, 0]
984 linearizer.linearityType[ampName] =
"LookupTable"
985 elif linearizerType ==
"LINEARIZESQUARED":
986 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
987 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
988 linearizer.linearityFitReducedChiSquared[ampName] = (
989 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
990 linearizer.linearityCoeffs[ampName] = [
991 datasetNonLinAmp.quadraticPolynomialLinearizerCoefficient]
992 linearizer.linearityType[ampName] =
"Squared"
993 elif linearizerType ==
"LINEARIZEPOLYNOMIAL":
994 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
995 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
996 linearizer.linearityFitReducedChiSquared[ampName] = (
997 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1001 polyLinCoeffs = np.array(datasetNonLinAmp.polynomialLinearizerCoefficients[2:])
1002 linearizer.linearityCoeffs[ampName] = polyLinCoeffs
1003 linearizer.linearityType[ampName] =
"Polynomial"
1004 linearizer.linearityBBox[ampName] = amp.getBBox()
1005 linearizer.validate()
1006 calibId = f
"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
1009 raftName = detName.split(
"_")[0]
1010 calibId += f
" raftName={raftName}"
1013 calibId += f
" raftName={raftname}"
1015 serial = detector.getSerial()
1016 linearizer.updateMetadata(instrumentName=instruName, detectorId=f
"{detNum}",
1017 calibId=calibId, serial=serial, detectorName=f
"{detName}")
1022 def _initialParsForPolynomial(order):
1024 pars = np.zeros(order, dtype=np.float)
1031 def _boundsForPolynomial(initialPars):
1032 lowers = [np.NINF
for p
in initialPars]
1033 uppers = [np.inf
for p
in initialPars]
1035 return (lowers, uppers)
1038 def _boundsForAstier(initialPars):
1039 lowers = [np.NINF
for p
in initialPars]
1040 uppers = [np.inf
for p
in initialPars]
1041 return (lowers, uppers)
1044 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
1045 """Return a boolean array to mask bad points.
1047 A linear function has a constant ratio, so find the median
1048 value of the ratios, and exclude the points that deviate
1049 from that by more than a factor of maxDeviationPositive/negative.
1050 Asymmetric deviations are supported as we expect the PTC to turn
1051 down as the flux increases, but sometimes it anomalously turns
1052 upwards just before turning over, which ruins the fits, so it
1053 is wise to be stricter about restricting positive outliers than
1056 Too high and points that are so bad that fit will fail will be included
1057 Too low and the non-linear points will be excluded, biasing the NL fit."""
1058 ratios = [b/a
for (a, b)
in zip(means, variances)]
1059 medianRatio = np.median(ratios)
1060 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
1063 maxDeviationPositive = abs(maxDeviationPositive)
1064 maxDeviationNegative = -1. * abs(maxDeviationNegative)
1066 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
1067 else False for r
in ratioDeviations])
1070 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
1072 nBad = Counter(array)[0]
1077 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
1080 array[array == 0] = substituteValue
1084 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
1086 Fit the photon transfer curve with either a polynomial of the order
1087 specified in the task config, or using the Astier approximation.
1089 Sigma clipping is performed iteratively for the fit, as well as an
1090 initial clipping of data points that are more than
1091 config.initialNonLinearityExclusionThreshold away from lying on a
1092 straight line. This other step is necessary because the photon transfer
1093 curve turns over catastrophically at very high flux (because saturation
1094 drops the variance to ~0) and these far outliers cause the initial fit
1095 to fail, meaning the sigma cannot be calculated to perform the
1100 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1101 The dataset containing the means, variances and exposure times
1104 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1105 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
1109 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1110 This is the same dataset as the input paramter, however, it has been modified
1111 to include information such as the fit vectors and the fit parameters. See
1112 the class `PhotonTransferCurveDatase`.
1115 def errFunc(p, x, y):
1116 return ptcFunc(p, x) - y
1118 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1119 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1121 for i, ampName
in enumerate(dataset.ampNames):
1122 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
1123 meanVecOriginal = np.array(dataset.rawMeans[ampName])
1124 varVecOriginal = np.array(dataset.rawVars[ampName])
1127 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
1128 (meanVecOriginal <= self.config.maxMeanSignal))
1131 self.config.initialNonLinearityExclusionThresholdPositive,
1132 self.config.initialNonLinearityExclusionThresholdNegative)
1133 mask = mask & goodPoints
1135 if ptcFitType ==
'EXPAPPROXIMATION':
1136 ptcFunc = funcAstier
1137 parsIniPtc = [-1e-9, 1.0, 10.]
1139 if ptcFitType ==
'POLYNOMIAL':
1140 ptcFunc = funcPolynomial
1146 while count <= maxIterationsPtcOutliers:
1150 meanTempVec = meanVecOriginal[mask]
1151 varTempVec = varVecOriginal[mask]
1152 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1158 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1159 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
1160 mask = mask & newMask
1162 nDroppedTotal = Counter(mask)[
False]
1163 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1166 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1168 dataset.visitMask[ampName] = mask
1170 meanVecFinal = meanVecOriginal[mask]
1171 varVecFinal = varVecOriginal[mask]
1173 if Counter(mask)[
False] > 0:
1174 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
1175 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1177 if (len(meanVecFinal) < len(parsIniPtc)):
1178 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1179 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1183 dataset.badAmps.append(ampName)
1184 dataset.gain[ampName] = np.nan
1185 dataset.gainErr[ampName] = np.nan
1186 dataset.noise[ampName] = np.nan
1187 dataset.noiseErr[ampName] = np.nan
1188 dataset.ptcFitPars[ampName] = np.nan
1189 dataset.ptcFitParsError[ampName] = np.nan
1190 dataset.ptcFitReducedChiSquared[ampName] = np.nan
1194 if self.config.doFitBootstrap:
1195 parsFit, parsFitErr, reducedChiSqPtc =
fitBootstrap(parsIniPtc, meanVecFinal,
1196 varVecFinal, ptcFunc)
1198 parsFit, parsFitErr, reducedChiSqPtc =
fitLeastSq(parsIniPtc, meanVecFinal,
1199 varVecFinal, ptcFunc)
1200 dataset.ptcFitPars[ampName] = parsFit
1201 dataset.ptcFitParsError[ampName] = parsFitErr
1202 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1204 if ptcFitType ==
'EXPAPPROXIMATION':
1205 ptcGain = parsFit[1]
1206 ptcGainErr = parsFitErr[1]
1207 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1208 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1209 if ptcFitType ==
'POLYNOMIAL':
1210 ptcGain = 1./parsFit[1]
1211 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1212 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1213 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1214 dataset.gain[ampName] = ptcGain
1215 dataset.gainErr[ampName] = ptcGainErr
1216 dataset.noise[ampName] = ptcNoise
1217 dataset.noiseErr[ampName] = ptcNoiseErr
1218 if not len(dataset.ptcFitType) == 0:
1219 dataset.ptcFitType = ptcFitType