32 from .astierCovPtcUtils
import (CovFastFourierTransform, computeCovDirect)
33 from .astierCovPtcFit
import makeCovArray
38 __all__ = [
'PhotonTransferCurveExtractConfig',
'PhotonTransferCurveExtractTask']
42 dimensions=(
"instrument",
"detector")):
45 name=
"ptcInputExposurePairs",
46 doc=
"Input post-ISR processed exposure pairs (flats) to"
47 "measure covariances from.",
48 storageClass=
"Exposure",
49 dimensions=(
"instrument",
"exposure",
"detector"),
54 outputCovariances = cT.Output(
55 name=
"ptcCovariances",
56 doc=
"Extracted flat (co)variances.",
57 storageClass=
"PhotonTransferCurveDataset",
58 dimensions=(
"instrument",
"exposure",
"detector"),
64 pipelineConnections=PhotonTransferCurveExtractConnections):
65 """Configuration for the measurement of covariances from flats.
67 matchByExposureId = pexConfig.Field(
69 doc=
"Should exposures by matched by ID rather than exposure time?",
72 maximumRangeCovariancesAstier = pexConfig.Field(
74 doc=
"Maximum range of covariances as in Astier+19",
77 covAstierRealSpace = pexConfig.Field(
79 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
82 binSize = pexConfig.Field(
84 doc=
"Bin the image by this factor in both dimensions.",
87 minMeanSignal = pexConfig.DictField(
90 doc=
"Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
91 " The same cut is applied to all amps if this dictionary is of the form"
92 " {'ALL_AMPS': value}",
93 default={
'ALL_AMPS': 0.0},
95 maxMeanSignal = pexConfig.DictField(
98 doc=
"Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
99 " The same cut is applied to all amps if this dictionary is of the form"
100 " {'ALL_AMPS': value}",
101 default={
'ALL_AMPS': 1e6},
103 maskNameList = pexConfig.ListField(
105 doc=
"Mask list to exclude from statistics calculations.",
106 default=[
'SUSPECT',
'BAD',
'NO_DATA',
'SAT'],
108 nSigmaClipPtc = pexConfig.Field(
110 doc=
"Sigma cut for afwMath.StatisticsControl()",
113 nIterSigmaClipPtc = pexConfig.Field(
115 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
118 minNumberGoodPixelsForCovariance = pexConfig.Field(
120 doc=
"Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
124 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
126 doc=
"If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
127 "for a region of a difference image is greater than this threshold (percentage), "
128 "a warning will be issued.",
131 detectorMeasurementRegion = pexConfig.ChoiceField(
133 doc=
"Region of each exposure where to perform the calculations (amplifier or full image).",
136 "AMP":
"Amplifier of the detector.",
137 "FULL":
"Full image."
140 numEdgeSuspect = pexConfig.Field(
142 doc=
"Number of edge pixels to be flagged as untrustworthy.",
145 edgeMaskLevel = pexConfig.ChoiceField(
147 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
150 'DETECTOR':
'Mask only the edges of the full detector.',
151 'AMP':
'Mask edges of each amplifier.',
157 pipeBase.CmdLineTask):
158 """Task to measure covariances from flat fields.
159 This task receives as input a list of flat-field images
160 (flats), and sorts these flats in pairs taken at the
161 same time (if there's a different number of flats,
162 those flats are discarded). The mean, variance, and
163 covariances are measured from the difference of the flat
164 pairs at a given time. The variance is calculated
165 via afwMath, and the covariance via the methods in Astier+19
166 (appendix A). In theory, var = covariance[0,0]. This should
167 be validated, and in the future, we may decide to just keep
170 The measured covariances at a particular time (along with
171 other quantities such as the mean) are stored in a PTC dataset
172 object (`PhotonTransferCurveDataset`), which gets partially
173 filled. The number of partially-filled PTC dataset objects
174 will be less than the number of input exposures, but gen3
175 requires/assumes that the number of input dimensions matches
176 bijectively the number of output dimensions. Therefore, a
177 number of "dummy" PTC dataset are inserted in the output list
178 that has the partially-filled PTC datasets with the covariances.
179 This output list will be used as input of
180 `PhotonTransferCurveSolveTask`, which will assemble the multiple
181 `PhotonTransferCurveDataset`s into a single one in order to fit
182 the measured covariances as a function of flux to a particular
185 Astier+19: "The Shape of the Photon Transfer Curve of CCD
186 sensors", arXiv:1905.08677.
188 ConfigClass = PhotonTransferCurveExtractConfig
189 _DefaultName =
'cpPtcExtract'
192 """Ensure that the input and output dimensions are passed along.
196 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
197 Butler to operate on.
198 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
199 Input data refs to load.
200 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
201 Output data refs to persist.
203 inputs = butlerQC.get(inputRefs)
205 if self.config.matchByExposureId:
210 inputs[
'inputDims'] = [expId.dataId[
'exposure']
for expId
in inputRefs.inputExp]
211 outputs = self.
runrun(**inputs)
212 butlerQC.put(outputs, outputRefs)
214 def run(self, inputExp, inputDims):
215 """Measure covariances from difference of flat pairs
219 inputExp : `dict` [`float`,
220 (`~lsst.afw.image.exposure.exposure.ExposureF`,
221 `~lsst.afw.image.exposure.exposure.ExposureF`, ...,
222 `~lsst.afw.image.exposure.exposure.ExposureF`)]
223 Dictionary that groups flat-field exposures that have the same
224 exposure time (seconds).
227 List of exposure IDs.
231 detector = list(inputExp.values())[0][0].getDetector()
232 detNum = detector.getId()
233 amps = detector.getAmplifiers()
234 ampNames = [amp.getName()
for amp
in amps]
237 maxMeanSignalDict = {ampName: 1e6
for ampName
in ampNames}
238 minMeanSignalDict = {ampName: 0.0
for ampName
in ampNames}
239 for ampName
in ampNames:
240 if 'ALL_AMPS' in self.config.maxMeanSignal:
241 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[
'ALL_AMPS']
242 elif ampName
in self.config.maxMeanSignal:
243 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
245 if 'ALL_AMPS' in self.config.minMeanSignal:
246 minMeanSignalDict[ampName] = self.config.minMeanSignal[
'ALL_AMPS']
247 elif ampName
in self.config.minMeanSignal:
248 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
250 tags = [(
'mu',
'<f8'), (
'afwVar',
'<f8'), (
'i',
'<i8'), (
'j',
'<i8'), (
'var',
'<f8'),
251 (
'cov',
'<f8'), (
'npix',
'<i8'), (
'ext',
'<i8'), (
'expTime',
'<f8'), (
'ampName',
'<U3')]
253 dummyPtcDataset = PhotonTransferCurveDataset(ampNames,
'DUMMY',
254 self.config.maximumRangeCovariancesAstier)
256 for ampName
in ampNames:
257 dummyPtcDataset.setAmpValues(ampName)
259 partialPtcDatasetList = []
262 for i
in range(len(inputDims)):
263 partialPtcDatasetList.append(dummyPtcDataset)
265 if self.config.numEdgeSuspect > 0:
267 self.log.info(f
"Masking {self.config.numEdgeSuspect} pixels from the edges "
268 "of all exposures as SUSPECT.")
270 for expTime
in inputExp:
271 exposures = inputExp[expTime]
272 if len(exposures) == 1:
273 self.log.warn(f
"Only one exposure found at expTime {expTime}. Dropping exposure "
274 f
"{exposures[0].getInfo().getVisitInfo().getExposureId()}.")
278 exp1, exp2 = exposures[0], exposures[1]
279 if len(exposures) > 2:
280 self.log.warn(f
"Already found 2 exposures at expTime {expTime}. "
281 "Ignoring exposures: "
282 f
"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}")
284 if self.config.numEdgeSuspect > 0:
285 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
286 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
287 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
288 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
289 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
290 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
292 partialPtcDataset = PhotonTransferCurveDataset(ampNames,
'',
293 self.config.maximumRangeCovariancesAstier)
294 for ampNumber, amp
in enumerate(detector):
295 ampName = amp.getName()
297 doRealSpace = self.config.covAstierRealSpace
298 if self.config.detectorMeasurementRegion ==
'AMP':
299 region = amp.getBBox()
300 elif self.config.detectorMeasurementRegion ==
'FULL':
306 muDiff, varDiff, covAstier = self.
measureMeanVarCovmeasureMeanVarCov(exp1, exp2, region=region,
307 covAstierRealSpace=doRealSpace)
314 if np.isnan(muDiff)
or np.isnan(varDiff)
or (covAstier
is None):
315 msg = (f
"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
316 f
" {expId2} of detector {detNum}.")
320 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
321 self.config.maximumRangeCovariancesAstier), np.nan)
322 covSqrtWeights = np.full_like(covArray, np.nan)
324 if (muDiff <= minMeanSignalDict[ampName])
or (muDiff >= maxMeanSignalDict[ampName]):
327 if covAstier
is not None:
328 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
329 ampName)
for covRow
in covAstier]
330 tempStructArray = np.array(tupleRows, dtype=tags)
332 self.config.maximumRangeCovariancesAstier)
333 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
337 covArray *= varFactor**2
339 covArray[0, 0] /= varFactor
341 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff],
342 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)],
343 expIdMask=[expIdMask], covArray=covArray,
344 covSqrtWeights=covSqrtWeights)
355 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
358 datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0]
360 datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0]
361 partialPtcDatasetList[datasetIndex] = partialPtcDataset
362 if nAmpsNan == len(ampNames):
363 msg = f
"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
365 return pipeBase.Struct(
366 outputCovariances=partialPtcDatasetList,
370 """Calculate the mean of each of two exposures and the variance
371 and covariance of their difference. The variance is calculated
372 via afwMath, and the covariance via the methods in Astier+19
373 (appendix A). In theory, var = covariance[0,0]. This should
374 be validated, and in the future, we may decide to just keep
379 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
380 First exposure of flat field pair.
381 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
382 Second exposure of flat field pair.
383 region : `lsst.geom.Box2I`, optional
384 Region of each exposure where to perform the calculations (e.g, an amplifier).
385 covAstierRealSpace : `bool`, optional
386 Should the covariannces in Astier+19 be calculated in real space or via FFT?
387 See Appendix A of Astier+19.
391 mu : `float` or `NaN`
392 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
393 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
394 varDiff : `float` or `NaN`
395 Half of the clipped variance of the difference of the regions inthe two input
396 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
397 covDiffAstier : `list` or `NaN`
398 List with tuples of the form (dx, dy, var, cov, npix), where:
404 Variance at (dx, dy).
406 Covariance at (dx, dy).
408 Number of pixel pairs used to evaluate var and cov.
409 If either mu1 or m2 are NaN's, the returned value is NaN.
412 if region
is not None:
413 im1Area = exposure1.maskedImage[region]
414 im2Area = exposure2.maskedImage[region]
416 im1Area = exposure1.maskedImage
417 im2Area = exposure2.maskedImage
419 if self.config.binSize > 1:
420 im1Area = afwMath.binImage(im1Area, self.config.binSize)
421 im2Area = afwMath.binImage(im2Area, self.config.binSize)
423 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
424 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
425 self.config.nIterSigmaClipPtc,
427 im1StatsCtrl.setNanSafe(
True)
428 im1StatsCtrl.setAndMask(im1MaskVal)
430 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
431 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
432 self.config.nIterSigmaClipPtc,
434 im2StatsCtrl.setNanSafe(
True)
435 im2StatsCtrl.setAndMask(im2MaskVal)
438 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
439 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
440 if np.isnan(mu1)
or np.isnan(mu2):
441 self.log.warn(f
"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
442 return np.nan, np.nan,
None
447 temp = im2Area.clone()
449 diffIm = im1Area.clone()
454 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
455 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
456 self.config.nIterSigmaClipPtc,
458 diffImStatsCtrl.setNanSafe(
True)
459 diffImStatsCtrl.setAndMask(diffImMaskVal)
462 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
466 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()
467 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, diffImStatsCtrl).getValue()
468 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
469 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
473 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
478 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance:
479 self.log.warn(f
"Number of good points for covariance calculation ({np.sum(w)}) is less "
480 f
"(than threshold {self.config.minNumberGoodPixelsForCovariance})")
481 return np.nan, np.nan,
None
483 maxRangeCov = self.config.maximumRangeCovariancesAstier
484 if covAstierRealSpace:
489 shapeDiff = np.array(diffIm.image.array.shape)
491 s = shapeDiff + maxRangeCov
492 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
493 fftSize = np.array(2**(tempSize+1)).astype(int)
494 fftShape = (fftSize[0], fftSize[1])
497 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
502 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
503 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
504 if fractionalDiff >= thresholdPercentage:
505 self.log.warn(
"Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
506 f
"is more than {thresholdPercentage}%: {fractionalDiff}")
508 return mu, varDiff, covDiffAstier
def makeCovArray(inputTuple, maxRangeFromTuple=8)
def computeCovDirect(diffImage, weightImage, maxRange)
def sigmaClipCorrection(nSigClip)
def arrangeFlatsByExpId(exposureList)
def arrangeFlatsByExpTime(exposureList)