23 __all__ = [
'MeasurePhotonTransferCurveTask',
24 'MeasurePhotonTransferCurveTaskConfig',
25 'PhotonTransferCurveDataset']
28 import matplotlib.pyplot
as plt
29 from sqlite3
import OperationalError
30 from collections
import Counter
31 from dataclasses
import dataclass
36 from .utils
import (NonexistentDatasetTaskDataIdContainer, PairedVisitListTaskRunner,
37 checkExpLengthEqual, fitLeastSq, fitBootstrap, funcPolynomial, funcAstier)
38 from scipy.optimize
import least_squares
43 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect, fitData)
47 """Config class for photon transfer curve measurement task"""
48 ccdKey = pexConfig.Field(
50 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'.",
53 ptcFitType = pexConfig.ChoiceField(
55 doc=
"Fit PTC to approximation in Astier+19 (Equation 16) or to a polynomial.",
58 "POLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegree' to set 'n').",
59 "EXPAPPROXIMATION":
"Approximation in Astier+19 (Eq. 16).",
60 "FULLCOVARIANCE":
"Full covariances model in Astier+19 (Eq. 20)"
63 sigmaClipFullFitCovariancesAstier = pexConfig.Field(
65 doc=
"sigma clip for full model fit for FULLCOVARIANCE ptcFitType ",
68 maxIterFullFitCovariancesAstier = pexConfig.Field(
70 doc=
"Maximum number of iterations in full model fit for FULLCOVARIANCE ptcFitType",
73 maximumRangeCovariancesAstier = pexConfig.Field(
75 doc=
"Maximum range of covariances as in Astier+19",
78 covAstierRealSpace = pexConfig.Field(
80 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
83 polynomialFitDegree = pexConfig.Field(
85 doc=
"Degree of polynomial to fit the PTC, when 'ptcFitType'=POLYNOMIAL.",
88 doCreateLinearizer = pexConfig.Field(
90 doc=
"Calculate non-linearity and persist linearizer?",
93 linearizerType = pexConfig.ChoiceField(
95 doc=
"Linearizer type, if doCreateLinearizer=True",
96 default=
"LINEARIZEPOLYNOMIAL",
98 "LINEARIZEPOLYNOMIAL":
"n-degree polynomial (use 'polynomialFitDegreeNonLinearity' to set 'n').",
99 "LINEARIZESQUARED":
"c0 quadratic coefficient derived from coefficients of polynomiual fit",
100 "LOOKUPTABLE":
"Loouk table formed from linear part of polynomial fit."
103 polynomialFitDegreeNonLinearity = pexConfig.Field(
105 doc=
"If doCreateLinearizer, degree of polynomial to fit the meanSignal vs exposureTime" +
106 " curve to produce the table for LinearizeLookupTable.",
109 binSize = pexConfig.Field(
111 doc=
"Bin the image by this factor in both dimensions.",
114 minMeanSignal = pexConfig.Field(
116 doc=
"Minimum value (inclusive) of mean signal (in DN) above which to consider.",
119 maxMeanSignal = pexConfig.Field(
121 doc=
"Maximum value (inclusive) of mean signal (in DN) below which to consider.",
124 initialNonLinearityExclusionThresholdPositive = pexConfig.RangeField(
126 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
127 " linear in the positive direction, from the PTC fit. Note that these points will also be"
128 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
129 " to allow an accurate determination of the sigmas for said iterative fit.",
134 initialNonLinearityExclusionThresholdNegative = pexConfig.RangeField(
136 doc=
"Initially exclude data points with a variance that are more than a factor of this from being"
137 " linear in the negative direction, from the PTC fit. Note that these points will also be"
138 " excluded from the non-linearity fit. This is done before the iterative outlier rejection,"
139 " to allow an accurate determination of the sigmas for said iterative fit.",
144 sigmaCutPtcOutliers = pexConfig.Field(
146 doc=
"Sigma cut for outlier rejection in PTC.",
149 maskNameList = pexConfig.ListField(
151 doc=
"Mask list to exclude from statistics calculations.",
152 default=[
'SUSPECT',
'BAD',
'NO_DATA'],
154 nSigmaClipPtc = pexConfig.Field(
156 doc=
"Sigma cut for afwMath.StatisticsControl()",
159 nIterSigmaClipPtc = pexConfig.Field(
161 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
164 maxIterationsPtcOutliers = pexConfig.Field(
166 doc=
"Maximum number of iterations for outlier rejection in PTC.",
169 doFitBootstrap = pexConfig.Field(
171 doc=
"Use bootstrap for the PTC fit parameters and errors?.",
174 maxAduForLookupTableLinearizer = pexConfig.Field(
176 doc=
"Maximum DN value for the LookupTable linearizer.",
179 instrumentName = pexConfig.Field(
181 doc=
"Instrument name.",
188 """A simple class to hold the output from the
189 `calculateLinearityResidualAndLinearizers` function.
192 polynomialLinearizerCoefficients: list
194 quadraticPolynomialLinearizerCoefficient: float
196 linearizerTableRow: list
197 meanSignalVsTimePolyFitPars: list
198 meanSignalVsTimePolyFitParsErr: list
199 meanSignalVsTimePolyFitReducedChiSq: float
203 """A simple class to hold the output data from the PTC task.
205 The dataset is made up of a dictionary for each item, keyed by the
206 amplifiers' names, which much be supplied at construction time.
208 New items cannot be added to the class to save accidentally saving to the
209 wrong property, and the class can be frozen if desired.
211 inputVisitPairs records the visits used to produce the data.
212 When fitPtc() or fitCovariancesAstier() is run, a mask is built up, which is by definition
213 always the same length as inputVisitPairs, rawExpTimes, rawMeans
214 and rawVars, and is a list of bools, which are incrementally set to False
215 as points are discarded from the fits.
217 PTC fit parameters for polynomials are stored in a list in ascending order
218 of polynomial term, i.e. par[0]*x^0 + par[1]*x + par[2]*x^2 etc
219 with the length of the list corresponding to the order of the polynomial
225 List with the names of the amplifiers of the detector at hand.
228 Type of model fitted to the PTC: "POLYNOMIAL", "EXPAPPROXIMATION", or "FULLCOVARIANCE".
232 `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
233 Output dataset from MeasurePhotonTransferCurveTask.
240 self.__dict__[
"ptcFitType"] = ptcFitType
241 self.__dict__[
"ampNames"] = ampNames
242 self.__dict__[
"badAmps"] = []
247 self.__dict__[
"inputVisitPairs"] = {ampName: []
for ampName
in ampNames}
248 self.__dict__[
"visitMask"] = {ampName: []
for ampName
in ampNames}
249 self.__dict__[
"rawExpTimes"] = {ampName: []
for ampName
in ampNames}
250 self.__dict__[
"rawMeans"] = {ampName: []
for ampName
in ampNames}
251 self.__dict__[
"rawVars"] = {ampName: []
for ampName
in ampNames}
254 self.__dict__[
"gain"] = {ampName: -1.
for ampName
in ampNames}
255 self.__dict__[
"gainErr"] = {ampName: -1.
for ampName
in ampNames}
256 self.__dict__[
"noise"] = {ampName: -1.
for ampName
in ampNames}
257 self.__dict__[
"noiseErr"] = {ampName: -1.
for ampName
in ampNames}
261 self.__dict__[
"ptcFitPars"] = {ampName: []
for ampName
in ampNames}
262 self.__dict__[
"ptcFitParsError"] = {ampName: []
for ampName
in ampNames}
263 self.__dict__[
"ptcFitReducedChiSquared"] = {ampName: []
for ampName
in ampNames}
270 self.__dict__[
"covariancesTuple"] = {ampName: []
for ampName
in ampNames}
271 self.__dict__[
"covariancesFitsWithNoB"] = {ampName: []
for ampName
in ampNames}
272 self.__dict__[
"covariancesFits"] = {ampName: []
for ampName
in ampNames}
273 self.__dict__[
"aMatrix"] = {ampName: []
for ampName
in ampNames}
274 self.__dict__[
"bMatrix"] = {ampName: []
for ampName
in ampNames}
277 self.__dict__[
"finalVars"] = {ampName: []
for ampName
in ampNames}
278 self.__dict__[
"finalModelVars"] = {ampName: []
for ampName
in ampNames}
279 self.__dict__[
"finalMeans"] = {ampName: []
for ampName
in ampNames}
282 """Protect class attributes"""
283 if attribute
not in self.__dict__:
284 raise AttributeError(f
"{attribute} is not already a member of PhotonTransferCurveDataset, which"
285 " does not support setting of new attributes.")
287 self.__dict__[attribute] = value
290 """Get the visits used, i.e. not discarded, for a given amp.
292 If no mask has been created yet, all visits are returned.
294 if len(self.visitMask[ampName]) == 0:
295 return self.inputVisitPairs[ampName]
298 assert len(self.visitMask[ampName]) == len(self.inputVisitPairs[ampName])
300 pairs = self.inputVisitPairs[ampName]
301 mask = self.visitMask[ampName]
303 return [(v1, v2)
for ((v1, v2), m)
in zip(pairs, mask)
if bool(m)
is True]
306 return [amp
for amp
in self.ampNames
if amp
not in self.badAmps]
310 """A class to calculate, fit, and plot a PTC from a set of flat pairs.
312 The Photon Transfer Curve (var(signal) vs mean(signal)) is a standard tool
313 used in astronomical detectors characterization (e.g., Janesick 2001,
314 Janesick 2007). If ptcFitType is "EXPAPPROXIMATION" or "POLYNOMIAL", this task calculates the
315 PTC from a series of pairs of flat-field images; each pair taken at identical exposure
316 times. The difference image of each pair is formed to eliminate fixed pattern noise,
317 and then the variance of the difference image and the mean of the average image
318 are used to produce the PTC. An n-degree polynomial or the approximation in Equation
319 16 of Astier+19 ("The Shape of the Photon Transfer Curve of CCD sensors",
320 arXiv:1905.08677) can be fitted to the PTC curve. These models include
321 parameters such as the gain (e/DN) and readout noise.
323 Linearizers to correct for signal-chain non-linearity are also calculated.
324 The `Linearizer` class, in general, can support per-amp linearizers, but in this
325 task this is not supported.
327 If ptcFitType is "FULLCOVARIANCE", the covariances of the difference images are calculated via the
328 DFT methods described in Astier+19 and the variances for the PTC are given by the cov[0,0] elements
329 at each signal level. The full model in Equation 20 of Astier+19 is fit to the PTC to get the gain
336 Positional arguments passed to the Task constructor. None used at this
339 Keyword arguments passed on to the Task constructor. None used at this
344 RunnerClass = PairedVisitListTaskRunner
345 ConfigClass = MeasurePhotonTransferCurveTaskConfig
346 _DefaultName =
"measurePhotonTransferCurve"
349 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
350 plt.interactive(
False)
351 self.config.validate()
355 def _makeArgumentParser(cls):
356 """Augment argument parser for the MeasurePhotonTransferCurveTask."""
358 parser.add_argument(
"--visit-pairs", dest=
"visitPairs", nargs=
"*",
359 help=
"Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
360 parser.add_id_argument(
"--id", datasetType=
"photonTransferCurveDataset",
361 ContainerClass=NonexistentDatasetTaskDataIdContainer,
362 help=
"The ccds to use, e.g. --id ccd=0..100")
367 """Run the Photon Transfer Curve (PTC) measurement task.
369 For a dataRef (which is each detector here),
370 and given a list of visit pairs (postISR) at different exposure times,
375 dataRef : list of lsst.daf.persistence.ButlerDataRef
376 dataRef for the detector for the visits to be fit.
378 visitPairs : `iterable` of `tuple` of `int`
379 Pairs of visit numbers to be processed together
383 detNum = dataRef.dataId[self.config.ccdKey]
384 detector = dataRef.get(
'camera')[dataRef.dataId[self.config.ccdKey]]
390 for name
in dataRef.getButler().getKeys(
'bias'):
391 if name
not in dataRef.dataId:
393 dataRef.dataId[name] = \
394 dataRef.getButler().queryMetadata(
'raw', [name], detector=detNum)[0]
395 except OperationalError:
398 amps = detector.getAmplifiers()
399 ampNames = [amp.getName()
for amp
in amps]
401 self.log.info(
'Measuring PTC using %s visits for detector %s' % (visitPairs, detector.getId()))
405 for (v1, v2)
in visitPairs:
407 dataRef.dataId[
'expId'] = v1
408 exp1 = dataRef.get(
"postISRCCD", immediate=
True)
409 dataRef.dataId[
'expId'] = v2
410 exp2 = dataRef.get(
"postISRCCD", immediate=
True)
411 del dataRef.dataId[
'expId']
414 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
417 for ampNumber, amp
in enumerate(detector):
418 ampName = amp.getName()
420 doRealSpace = self.config.covAstierRealSpace
421 muDiff, varDiff, covAstier = self.
measureMeanVarCov(exp1, exp2, region=amp.getBBox(),
422 covAstierRealSpace=doRealSpace)
423 if np.isnan(muDiff)
or np.isnan(varDiff)
or (covAstier
is None):
424 msg = (f
"NaN mean or var, or None cov in amp {ampNumber} in visit pair {v1}, {v2} "
425 "of detector {detNum}.")
429 tags = [
'mu',
'i',
'j',
'var',
'cov',
'npix',
'ext',
'expTime',
'ampName']
430 if (muDiff <= self.config.minMeanSignal)
or (muDiff >= self.config.maxMeanSignal):
432 datasetPtc.rawExpTimes[ampName].append(expTime)
433 datasetPtc.rawMeans[ampName].append(muDiff)
434 datasetPtc.rawVars[ampName].append(varDiff)
435 datasetPtc.inputVisitPairs[ampName].append((v1, v2))
437 tupleRows += [(muDiff, ) + covRow + (ampNumber, expTime, ampName)
for covRow
in covAstier]
438 if nAmpsNan == len(ampNames):
439 msg = f
"NaN mean in all amps of visit pair {v1}, {v2} of detector {detNum}."
443 tupleRecords += tupleRows
444 covariancesWithTags = np.core.records.fromrecords(tupleRecords, names=allTags)
446 if self.config.ptcFitType
in [
"FULLCOVARIANCE", ]:
449 elif self.config.ptcFitType
in [
"EXPAPPROXIMATION",
"POLYNOMIAL"]:
452 datasetPtc = self.
fitPtc(datasetPtc, self.config.ptcFitType)
455 if self.config.doCreateLinearizer:
456 numberAmps = len(amps)
457 numberAduValues = self.config.maxAduForLookupTableLinearizer
458 lookupTableArray = np.zeros((numberAmps, numberAduValues), dtype=np.float32)
466 tableArray=lookupTableArray,
469 if self.config.linearizerType ==
"LINEARIZEPOLYNOMIAL":
470 linDataType =
'linearizePolynomial'
471 linMsg =
"polynomial (coefficients for a polynomial correction)."
472 elif self.config.linearizerType ==
"LINEARIZESQUARED":
473 linDataType =
'linearizePolynomial'
474 linMsg =
"squared (c0, derived from k_i coefficients of a polynomial fit)."
475 elif self.config.linearizerType ==
"LOOKUPTABLE":
476 linDataType =
'linearizePolynomial'
477 linMsg =
"lookup table (linear component of polynomial fit)."
479 raise RuntimeError(
"Invalid config.linearizerType {selg.config.linearizerType}. "
480 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
482 butler = dataRef.getButler()
483 self.log.info(f
"Writing linearizer: \n {linMsg}")
485 detName = detector.getName()
486 now = datetime.datetime.utcnow()
487 calibDate = now.strftime(
"%Y-%m-%d")
489 butler.put(linearizer, datasetType=linDataType, dataId={
'detector': detNum,
490 'detectorName': detName,
'calibDate': calibDate})
492 self.log.info(f
"Writing PTC data to {dataRef.getUri(write=True)}")
493 dataRef.put(datasetPtc, datasetType=
"photonTransferCurveDataset")
495 return pipeBase.Struct(exitStatus=0)
498 """Fit measured flat covariances to full model in Astier+19.
502 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
503 The dataset containing information such as the means, variances and exposure times.
505 covariancesWithTagsArray : `numpy.recarray`
506 Tuple with at least (mu, cov, var, i, j, npix), where:
507 mu : 0.5*(m1 + m2), where:
508 mu1: mean value of flat1
509 mu2: mean value of flat2
510 cov: covariance value at lag(i, j)
511 var: variance(covariance value at lag(0, 0))
514 npix: number of pixels used for covariance calculation.
518 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
519 This is the same dataset as the input paramter, however, it has been modified
520 to include information such as the fit vectors and the fit parameters. See
521 the class `PhotonTransferCurveDatase`.
524 covFits, covFitsNoB =
fitData(covariancesWithTagsArray, maxMu=self.config.maxMeanSignal,
525 r=self.config.maximumRangeCovariancesAstier,
526 nSigmaFullFit=self.config.sigmaClipFullFitCovariancesAstier,
527 maxIterFullFit=self.config.maxIterFullFitCovariancesAstier)
529 dataset.covariancesTuple = covariancesWithTagsArray
530 dataset.covariancesFits = covFits
531 dataset.covariancesFitsWithNoB = covFitsNoB
537 """Get output data for PhotonTransferCurveCovAstierDataset from CovFit objects.
541 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
542 The dataset containing information such as the means, variances and exposure times.
545 Dictionary of CovFit objects, with amp names as keys.
549 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
550 This is the same dataset as the input paramter, however, it has been modified
551 to include extra information such as the mask 1D array, gains, reoudout noise, measured signal,
552 measured variance, modeled variance, a, and b coefficient matrices (see Astier+19) per amplifier.
553 See the class `PhotonTransferCurveDatase`.
556 for i, amp
in enumerate(covFits):
558 (meanVecFinal, varVecFinal, varVecModel,
559 wc, varMask) = fit.getFitData(0, 0, divideByMu=
False, returnMasked=
True)
561 dataset.visitMask[amp] = varMask
562 dataset.gain[amp] = gain
563 dataset.gainErr[amp] = fit.getGainErr()
564 dataset.noise[amp] = np.sqrt(np.fabs(fit.getRon()))
565 dataset.noiseErr[amp] = fit.getRonErr()
566 dataset.finalVars[amp].append(varVecFinal/(gain**2))
567 dataset.finalModelVars[amp].append(varVecModel/(gain**2))
568 dataset.finalMeans[amp].append(meanVecFinal/gain)
569 dataset.aMatrix[amp].append(fit.getA())
570 dataset.bMatrix[amp].append(fit.getB())
575 """Calculate the mean of each of two exposures and the variance and covariance of their difference.
577 The variance is calculated via afwMath, and the covariance via the methods in Astier+19 (appendix A).
578 In theory, var = covariance[0,0]. This should be validated, and in the future, we may decide to just
579 keep one (covariance).
583 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
584 First exposure of flat field pair.
586 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
587 Second exposure of flat field pair.
589 region : `lsst.geom.Box2I`, optional
590 Region of each exposure where to perform the calculations (e.g, an amplifier).
592 covAstierRealSpace : `bool`, optional
593 Should the covariannces in Astier+19 be calculated in real space or via FFT?
594 See Appendix A of Astier+19.
598 mu : `float` or `NaN`
599 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
600 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
602 varDiff : `float` or `NaN`
603 Half of the clipped variance of the difference of the regions inthe two input
604 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
606 covDiffAstier : `list` or `NaN`
607 List with tuples of the form (dx, dy, var, cov, npix), where:
613 Variance at (dx, dy).
615 Covariance at (dx, dy).
617 Number of pixel pairs used to evaluate var and cov.
618 If either mu1 or m2 are NaN's, the returned value is NaN.
621 if region
is not None:
622 im1Area = exposure1.maskedImage[region]
623 im2Area = exposure2.maskedImage[region]
625 im1Area = exposure1.maskedImage
626 im2Area = exposure2.maskedImage
628 if self.config.binSize > 1:
629 im1Area = afwMath.binImage(im1Area, self.config.binSize)
630 im2Area = afwMath.binImage(im2Area, self.config.binSize)
632 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
633 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
634 self.config.nIterSigmaClipPtc,
636 im1StatsCtrl.setNanSafe(
True)
637 im1StatsCtrl.setAndMask(im1MaskVal)
639 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
640 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
641 self.config.nIterSigmaClipPtc,
643 im2StatsCtrl.setNanSafe(
True)
644 im2StatsCtrl.setAndMask(im2MaskVal)
647 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
648 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
649 if np.isnan(mu1)
or np.isnan(mu2):
650 return np.nan, np.nan,
None
655 temp = im2Area.clone()
657 diffIm = im1Area.clone()
662 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
663 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
664 self.config.nIterSigmaClipPtc,
666 diffImStatsCtrl.setNanSafe(
True)
667 diffImStatsCtrl.setAndMask(diffImMaskVal)
669 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
672 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
673 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
676 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
679 maxRangeCov = self.config.maximumRangeCovariancesAstier
680 if covAstierRealSpace:
681 covDiffAstier =
computeCovDirect(diffIm.getImage().getArray(), w, maxRangeCov)
683 shapeDiff = diffIm.getImage().getArray().shape
684 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
685 c =
CovFft(diffIm.getImage().getArray(), w, fftShape, maxRangeCov)
686 covDiffAstier = c.reportCovFft(maxRangeCov)
688 return mu, varDiff, covDiffAstier
691 """Compute covariances of diffImage in real space.
693 For lags larger than ~25, it is slower than the FFT way.
694 Taken from https://github.com/PierreAstier/bfptc/
698 diffImage : `numpy.array`
699 Image to compute the covariance of.
701 weightImage : `numpy.array`
702 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
705 Last index of the covariance to be computed.
710 List with tuples of the form (dx, dy, var, cov, npix), where:
716 Variance at (dx, dy).
718 Covariance at (dx, dy).
720 Number of pixel pairs used to evaluate var and cov.
725 for dy
in range(maxRange + 1):
726 for dx
in range(0, maxRange + 1):
729 cov2, nPix2 = self.
covDirectValue(diffImage, weightImage, dx, -dy)
730 cov = 0.5*(cov1 + cov2)
734 if (dx == 0
and dy == 0):
736 outList.append((dx, dy, var, cov, nPix))
741 """Compute covariances of diffImage in real space at lag (dx, dy).
743 Taken from https://github.com/PierreAstier/bfptc/ (c.f., appendix of Astier+19).
747 diffImage : `numpy.array`
748 Image to compute the covariance of.
750 weightImage : `numpy.array`
751 Weight image of diffImage (1's and 0's for good and bad pixels, respectively).
762 Covariance at (dx, dy)
765 Number of pixel pairs used to evaluate var and cov.
767 (nCols, nRows) = diffImage.shape
771 (dx, dy) = (-dx, -dy)
775 im1 = diffImage[dy:, dx:]
776 w1 = weightImage[dy:, dx:]
777 im2 = diffImage[:nCols - dy, :nRows - dx]
778 w2 = weightImage[:nCols - dy, :nRows - dx]
780 im1 = diffImage[:nCols + dy, dx:]
781 w1 = weightImage[:nCols + dy, dx:]
782 im2 = diffImage[-dy:, :nRows - dx]
783 w2 = weightImage[-dy:, :nRows - dx]
789 s1 = im1TimesW.sum()/nPix
790 s2 = (im2*wAll).sum()/nPix
791 p = (im1TimesW*im2).sum()/nPix
797 """Fit non-linearity function and build linearizer objects.
801 datasePtct : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
802 The dataset containing information such as the means, variances and exposure times.
805 detector : `lsst.afw.cameraGeom.Detector`
808 tableArray : `np.array`, optional
809 Optional. Look-up table array with size rows=nAmps and columns=DN values.
810 It will be modified in-place if supplied.
812 log : `lsst.log.Log`, optional
813 Logger to handle messages.
817 linearizer : `lsst.ip.isr.Linearizer`
822 datasetNonLinearity = self.
fitNonLinearity(datasetPtc, tableArray=tableArray)
825 now = datetime.datetime.utcnow()
826 calibDate = now.strftime(
"%Y-%m-%d")
827 linType = self.config.linearizerType
829 if linType ==
"LOOKUPTABLE":
830 tableArray = tableArray
835 instruName=self.config.instrumentName,
836 tableArray=tableArray,
842 """Fit a polynomial to signal vs effective time curve to calculate linearity and residuals.
846 datasetPtc : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
847 The dataset containing the means, variances and exposure times.
849 tableArray : `np.array`
850 Optional. Look-up table array with size rows=nAmps and columns=DN values.
851 It will be modified in-place if supplied.
855 datasetNonLinearity : `dict`
856 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
857 dataclasses. Each one holds the output of `calculateLinearityResidualAndLinearizers` per
860 datasetNonLinearity = {ampName: []
for ampName
in datasetPtc.ampNames}
861 for i, ampName
in enumerate(datasetPtc.ampNames):
863 if (len(datasetPtc.visitMask[ampName]) == 0):
864 self.log.warn(f
"Mask not found for {ampName} in non-linearity fit. Using all points.")
865 mask = np.repeat(
True, len(datasetPtc.rawExpTimes[ampName]))
867 mask = datasetPtc.visitMask[ampName]
869 timeVecFinal = np.array(datasetPtc.rawExpTimes[ampName])[mask]
870 meanVecFinal = np.array(datasetPtc.rawMeans[ampName])[mask]
877 if tableArray
is not None:
878 tableArray[i, :] = datasetLinRes.linearizerTableRow
880 datasetNonLinearity[ampName] = datasetLinRes
882 return datasetNonLinearity
885 """Calculate linearity residual and fit an n-order polynomial to the mean vs time curve
886 to produce corrections (deviation from linear part of polynomial) for a particular amplifier
887 to populate LinearizeLookupTable.
888 Use the coefficients of this fit to calculate the correction coefficients for LinearizePolynomial
889 and LinearizeSquared."
894 exposureTimeVector: `list` of `float`
895 List of exposure times for each flat pair
897 meanSignalVector: `list` of `float`
898 List of mean signal from diference image of flat pairs
902 dataset : `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset`
903 The dataset containing the fit parameters, the NL correction coefficients, and the
904 LUT row for the amplifier at hand.
910 dataset.polynomialLinearizerCoefficients : `list` of `float`
911 Coefficients for LinearizePolynomial, where corrImage = uncorrImage + sum_i c_i uncorrImage^(2 +
913 c_(j-2) = -k_j/(k_1^j) with units DN^(1-j) (c.f., Eq. 37 of 2003.05978). The units of k_j are
914 DN/t^j, and they are fit from meanSignalVector = k0 + k1*exposureTimeVector +
915 k2*exposureTimeVector^2 + ... + kn*exposureTimeVector^n, with
916 n = "polynomialFitDegreeNonLinearity". k_0 and k_1 and degenerate with bias level and gain,
917 and are not used by the non-linearity correction. Therefore, j = 2...n in the above expression
918 (see `LinearizePolynomial` class in `linearize.py`.)
920 dataset.quadraticPolynomialLinearizerCoefficient : `float`
921 Coefficient for LinearizeSquared, where corrImage = uncorrImage + c0*uncorrImage^2.
922 c0 = -k2/(k1^2), where k1 and k2 are fit from
923 meanSignalVector = k0 + k1*exposureTimeVector + k2*exposureTimeVector^2 +...
924 + kn*exposureTimeVector^n, with n = "polynomialFitDegreeNonLinearity".
926 dataset.linearizerTableRow : `list` of `float`
927 One dimensional array with deviation from linear part of n-order polynomial fit
928 to mean vs time curve. This array will be one row (for the particular amplifier at hand)
929 of the table array for LinearizeLookupTable.
931 dataset.meanSignalVsTimePolyFitPars : `list` of `float`
932 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
934 dataset.meanSignalVsTimePolyFitParsErr : `list` of `float`
935 Parameters from n-order polynomial fit to meanSignalVector vs exposureTimeVector.
937 dataset.meanSignalVsTimePolyFitReducedChiSq : `float`
938 Reduced unweighted chi squared from polynomial fit to meanSignalVector vs exposureTimeVector.
943 if self.config.doFitBootstrap:
944 (parsFit, parsFitErr,
945 reducedChiSquaredNonLinearityFit) =
fitBootstrap(parsIniNonLinearity,
949 weightsY=1./np.sqrt(meanSignalVector))
951 (parsFit, parsFitErr,
952 reducedChiSquaredNonLinearityFit) =
fitLeastSq(parsIniNonLinearity,
956 weightsY=1./np.sqrt(meanSignalVector))
960 tMax = (self.config.maxAduForLookupTableLinearizer - parsFit[0])/parsFit[1]
961 timeRange = np.linspace(0, tMax, self.config.maxAduForLookupTableLinearizer)
962 signalIdeal = parsFit[0] + parsFit[1]*timeRange
964 linearizerTableRow = signalIdeal - signalUncorrected
970 polynomialLinearizerCoefficients = []
971 for i, coefficient
in enumerate(parsFit):
972 c = -coefficient/(k1**i)
973 polynomialLinearizerCoefficients.append(c)
974 if np.fabs(c) > 1e-10:
975 msg = f
"Coefficient {c} in polynomial fit larger than threshold 1e-10."
978 c0 = polynomialLinearizerCoefficients[2]
981 dataset.polynomialLinearizerCoefficients = polynomialLinearizerCoefficients
982 dataset.quadraticPolynomialLinearizerCoefficient = c0
983 dataset.linearizerTableRow = linearizerTableRow
984 dataset.meanSignalVsTimePolyFitPars = parsFit
985 dataset.meanSignalVsTimePolyFitParsErr = parsFitErr
986 dataset.meanSignalVsTimePolyFitReducedChiSq = reducedChiSquaredNonLinearityFit
991 tableArray=None, log=None):
992 """Build linearizer object to persist.
996 datasetNonLinearity : `dict`
997 Dictionary of `lsst.cp.pipe.ptc.LinearityResidualsAndLinearizersDataset` objects.
999 detector : `lsst.afw.cameraGeom.Detector`
1002 calibDate : `datetime.datetime`
1005 linearizerType : `str`
1006 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'
1008 instruName : `str`, optional
1011 tableArray : `np.array`, optional
1012 Look-up table array with size rows=nAmps and columns=DN values
1014 log : `lsst.log.Log`, optional
1015 Logger to handle messages
1019 linearizer : `lsst.ip.isr.Linearizer`
1022 detName = detector.getName()
1023 detNum = detector.getId()
1024 if linearizerType ==
"LOOKUPTABLE":
1025 if tableArray
is not None:
1026 linearizer =
Linearizer(detector=detector, table=tableArray, log=log)
1028 raise RuntimeError(
"tableArray must be provided when creating a LookupTable linearizer")
1029 elif linearizerType
in (
"LINEARIZESQUARED",
"LINEARIZEPOLYNOMIAL"):
1032 raise RuntimeError(
"Invalid linearizerType {linearizerType} to build a Linearizer object. "
1033 "Supported: 'LOOKUPTABLE', 'LINEARIZESQUARED', or 'LINEARIZEPOLYNOMIAL'")
1034 for i, amp
in enumerate(detector.getAmplifiers()):
1035 ampName = amp.getName()
1036 datasetNonLinAmp = datasetNonLinearity[ampName]
1037 if linearizerType ==
"LOOKUPTABLE":
1038 linearizer.linearityCoeffs[ampName] = [i, 0]
1039 linearizer.linearityType[ampName] =
"LookupTable"
1040 elif linearizerType ==
"LINEARIZESQUARED":
1041 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1042 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1043 linearizer.linearityFitReducedChiSquared[ampName] = (
1044 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1045 linearizer.linearityCoeffs[ampName] = [
1046 datasetNonLinAmp.quadraticPolynomialLinearizerCoefficient]
1047 linearizer.linearityType[ampName] =
"Squared"
1048 elif linearizerType ==
"LINEARIZEPOLYNOMIAL":
1049 linearizer.fitParams[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitPars
1050 linearizer.fitParamsErr[ampName] = datasetNonLinAmp.meanSignalVsTimePolyFitParsErr
1051 linearizer.linearityFitReducedChiSquared[ampName] = (
1052 datasetNonLinAmp.meanSignalVsTimePolyFitReducedChiSq)
1056 polyLinCoeffs = np.array(datasetNonLinAmp.polynomialLinearizerCoefficients[2:])
1057 linearizer.linearityCoeffs[ampName] = polyLinCoeffs
1058 linearizer.linearityType[ampName] =
"Polynomial"
1059 linearizer.linearityBBox[ampName] = amp.getBBox()
1060 linearizer.validate()
1061 calibId = f
"detectorName={detName} detector={detNum} calibDate={calibDate} ccd={detNum} filter=NONE"
1064 raftName = detName.split(
"_")[0]
1065 calibId += f
" raftName={raftName}"
1068 calibId += f
" raftName={raftname}"
1070 serial = detector.getSerial()
1071 linearizer.updateMetadata(instrumentName=instruName, detectorId=f
"{detNum}",
1072 calibId=calibId, serial=serial, detectorName=f
"{detName}")
1077 def _initialParsForPolynomial(order):
1079 pars = np.zeros(order, dtype=np.float)
1086 def _boundsForPolynomial(initialPars):
1087 lowers = [np.NINF
for p
in initialPars]
1088 uppers = [np.inf
for p
in initialPars]
1090 return (lowers, uppers)
1093 def _boundsForAstier(initialPars):
1094 lowers = [np.NINF
for p
in initialPars]
1095 uppers = [np.inf
for p
in initialPars]
1096 return (lowers, uppers)
1099 def _getInitialGoodPoints(means, variances, maxDeviationPositive, maxDeviationNegative):
1100 """Return a boolean array to mask bad points.
1102 A linear function has a constant ratio, so find the median
1103 value of the ratios, and exclude the points that deviate
1104 from that by more than a factor of maxDeviationPositive/negative.
1105 Asymmetric deviations are supported as we expect the PTC to turn
1106 down as the flux increases, but sometimes it anomalously turns
1107 upwards just before turning over, which ruins the fits, so it
1108 is wise to be stricter about restricting positive outliers than
1111 Too high and points that are so bad that fit will fail will be included
1112 Too low and the non-linear points will be excluded, biasing the NL fit."""
1113 ratios = [b/a
for (a, b)
in zip(means, variances)]
1114 medianRatio = np.median(ratios)
1115 ratioDeviations = [(r/medianRatio)-1
for r
in ratios]
1118 maxDeviationPositive = abs(maxDeviationPositive)
1119 maxDeviationNegative = -1. * abs(maxDeviationNegative)
1121 goodPoints = np.array([
True if (r < maxDeviationPositive
and r > maxDeviationNegative)
1122 else False for r
in ratioDeviations])
1125 def _makeZeroSafe(self, array, warn=True, substituteValue=1e-9):
1127 nBad = Counter(array)[0]
1132 msg = f
"Found {nBad} zeros in array at elements {[x for x in np.where(array==0)[0]]}"
1135 array[array == 0] = substituteValue
1139 """Fit the photon transfer curve to a polynimial or to Astier+19 approximation.
1141 Fit the photon transfer curve with either a polynomial of the order
1142 specified in the task config, or using the Astier approximation.
1144 Sigma clipping is performed iteratively for the fit, as well as an
1145 initial clipping of data points that are more than
1146 config.initialNonLinearityExclusionThreshold away from lying on a
1147 straight line. This other step is necessary because the photon transfer
1148 curve turns over catastrophically at very high flux (because saturation
1149 drops the variance to ~0) and these far outliers cause the initial fit
1150 to fail, meaning the sigma cannot be calculated to perform the
1155 dataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1156 The dataset containing the means, variances and exposure times
1159 Fit a 'POLYNOMIAL' (degree: 'polynomialFitDegree') or
1160 'EXPAPPROXIMATION' (Eq. 16 of Astier+19) to the PTC
1164 dataset: `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
1165 This is the same dataset as the input paramter, however, it has been modified
1166 to include information such as the fit vectors and the fit parameters. See
1167 the class `PhotonTransferCurveDatase`.
1170 def errFunc(p, x, y):
1171 return ptcFunc(p, x) - y
1173 sigmaCutPtcOutliers = self.config.sigmaCutPtcOutliers
1174 maxIterationsPtcOutliers = self.config.maxIterationsPtcOutliers
1176 for i, ampName
in enumerate(dataset.ampNames):
1177 timeVecOriginal = np.array(dataset.rawExpTimes[ampName])
1178 meanVecOriginal = np.array(dataset.rawMeans[ampName])
1179 varVecOriginal = np.array(dataset.rawVars[ampName])
1182 mask = ((meanVecOriginal >= self.config.minMeanSignal) &
1183 (meanVecOriginal <= self.config.maxMeanSignal))
1186 self.config.initialNonLinearityExclusionThresholdPositive,
1187 self.config.initialNonLinearityExclusionThresholdNegative)
1188 mask = mask & goodPoints
1190 if ptcFitType ==
'EXPAPPROXIMATION':
1191 ptcFunc = funcAstier
1192 parsIniPtc = [-1e-9, 1.0, 10.]
1194 if ptcFitType ==
'POLYNOMIAL':
1195 ptcFunc = funcPolynomial
1201 while count <= maxIterationsPtcOutliers:
1205 meanTempVec = meanVecOriginal[mask]
1206 varTempVec = varVecOriginal[mask]
1207 res = least_squares(errFunc, parsIniPtc, bounds=bounds, args=(meanTempVec, varTempVec))
1213 sigResids = (varVecOriginal - ptcFunc(pars, meanVecOriginal))/np.sqrt(varVecOriginal)
1214 newMask = np.array([
True if np.abs(r) < sigmaCutPtcOutliers
else False for r
in sigResids])
1215 mask = mask & newMask
1217 nDroppedTotal = Counter(mask)[
False]
1218 self.log.debug(f
"Iteration {count}: discarded {nDroppedTotal} points in total for {ampName}")
1221 assert (len(mask) == len(timeVecOriginal) == len(meanVecOriginal) == len(varVecOriginal))
1223 dataset.visitMask[ampName] = mask
1225 meanVecFinal = meanVecOriginal[mask]
1226 varVecFinal = varVecOriginal[mask]
1228 if Counter(mask)[
False] > 0:
1229 self.log.info((f
"Number of points discarded in PTC of amplifier {ampName}:" +
1230 f
" {Counter(mask)[False]} out of {len(meanVecOriginal)}"))
1232 if (len(meanVecFinal) < len(parsIniPtc)):
1233 msg = (f
"\nSERIOUS: Not enough data points ({len(meanVecFinal)}) compared to the number of"
1234 f
"parameters of the PTC model({len(parsIniPtc)}). Setting {ampName} to BAD.")
1238 dataset.badAmps.append(ampName)
1239 dataset.gain[ampName] = np.nan
1240 dataset.gainErr[ampName] = np.nan
1241 dataset.noise[ampName] = np.nan
1242 dataset.noiseErr[ampName] = np.nan
1243 dataset.ptcFitPars[ampName] = np.nan
1244 dataset.ptcFitParsError[ampName] = np.nan
1245 dataset.ptcFitReducedChiSquared[ampName] = np.nan
1249 if self.config.doFitBootstrap:
1250 parsFit, parsFitErr, reducedChiSqPtc =
fitBootstrap(parsIniPtc, meanVecFinal,
1251 varVecFinal, ptcFunc,
1252 weightsY=1./np.sqrt(varVecFinal))
1254 parsFit, parsFitErr, reducedChiSqPtc =
fitLeastSq(parsIniPtc, meanVecFinal,
1255 varVecFinal, ptcFunc,
1256 weightsY=1./np.sqrt(varVecFinal))
1257 dataset.ptcFitPars[ampName] = parsFit
1258 dataset.ptcFitParsError[ampName] = parsFitErr
1259 dataset.ptcFitReducedChiSquared[ampName] = reducedChiSqPtc
1261 if ptcFitType ==
'EXPAPPROXIMATION':
1262 ptcGain = parsFit[1]
1263 ptcGainErr = parsFitErr[1]
1264 ptcNoise = np.sqrt(np.fabs(parsFit[2]))
1265 ptcNoiseErr = 0.5*(parsFitErr[2]/np.fabs(parsFit[2]))*np.sqrt(np.fabs(parsFit[2]))
1266 if ptcFitType ==
'POLYNOMIAL':
1267 ptcGain = 1./parsFit[1]
1268 ptcGainErr = np.fabs(1./parsFit[1])*(parsFitErr[1]/parsFit[1])
1269 ptcNoise = np.sqrt(np.fabs(parsFit[0]))*ptcGain
1270 ptcNoiseErr = (0.5*(parsFitErr[0]/np.fabs(parsFit[0]))*(np.sqrt(np.fabs(parsFit[0]))))*ptcGain
1271 dataset.gain[ampName] = ptcGain
1272 dataset.gainErr[ampName] = ptcGainErr
1273 dataset.noise[ampName] = ptcNoise
1274 dataset.noiseErr[ampName] = ptcNoiseErr
1275 if not len(dataset.ptcFitType) == 0:
1276 dataset.ptcFitType = ptcFitType