31 from .astierCovPtcUtils
import (fftSize, CovFft, computeCovDirect)
32 from .astierCovPtcFit
import makeCovArray
37 __all__ = [
'PhotonTransferCurveExtractConfig',
'PhotonTransferCurveExtractTask']
41 dimensions=(
"instrument",
"detector")):
44 name=
"ptcInputExposurePairs",
45 doc=
"Input post-ISR processed exposure pairs (flats) to"
46 "measure covariances from.",
47 storageClass=
"Exposure",
48 dimensions=(
"instrument",
"exposure",
"detector"),
53 outputCovariances = cT.Output(
54 name=
"ptcCovariances",
55 doc=
"Extracted flat (co)variances.",
56 storageClass=
"PhotonTransferCurveDataset",
57 dimensions=(
"instrument",
"exposure",
"detector"),
63 pipelineConnections=PhotonTransferCurveExtractConnections):
64 """Configuration for the measurement of covariances from flats.
66 maximumRangeCovariancesAstier = pexConfig.Field(
68 doc=
"Maximum range of covariances as in Astier+19",
71 covAstierRealSpace = pexConfig.Field(
73 doc=
"Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
76 binSize = pexConfig.Field(
78 doc=
"Bin the image by this factor in both dimensions.",
81 minMeanSignal = pexConfig.DictField(
84 doc=
"Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
85 " The same cut is applied to all amps if this dictionary is of the form"
86 " {'ALL_AMPS': value}",
87 default={
'ALL_AMPS': 0.0},
89 maxMeanSignal = pexConfig.DictField(
92 doc=
"Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
93 " The same cut is applied to all amps if this dictionary is of the form"
94 " {'ALL_AMPS': value}",
95 default={
'ALL_AMPS': 1e6},
97 maskNameList = pexConfig.ListField(
99 doc=
"Mask list to exclude from statistics calculations.",
100 default=[
'SUSPECT',
'BAD',
'NO_DATA'],
102 nSigmaClipPtc = pexConfig.Field(
104 doc=
"Sigma cut for afwMath.StatisticsControl()",
107 nIterSigmaClipPtc = pexConfig.Field(
109 doc=
"Number of sigma-clipping iterations for afwMath.StatisticsControl()",
112 minNumberGoodPixelsForFft = pexConfig.Field(
114 doc=
"Minimum number of acceptable good pixels per amp to calculate the covariances via FFT.",
117 detectorMeasurementRegion = pexConfig.ChoiceField(
119 doc=
"Region of each exposure where to perform the calculations (amplifier or full image).",
122 "AMP":
"Amplifier of the detector.",
123 "FULL":
"Full image."
129 pipeBase.CmdLineTask):
130 """Task to measure covariances from flat fields.
131 This task receives as input a list of flat-field images
132 (flats), and sorts these flats in pairs taken at the
133 same time (if there's a different number of flats,
134 those flats are discarded). The mean, variance, and
135 covariances are measured from the difference of the flat
136 pairs at a given time. The variance is calculated
137 via afwMath, and the covariance via the methods in Astier+19
138 (appendix A). In theory, var = covariance[0,0]. This should
139 be validated, and in the future, we may decide to just keep
142 The measured covariances at a particular time (along with
143 other quantities such as the mean) are stored in a PTC dataset
144 object (`PhotonTransferCurveDataset`), which gets partially
145 filled. The number of partially-filled PTC dataset objects
146 will be less than the number of input exposures, but gen3
147 requires/assumes that the number of input dimensions matches
148 bijectively the number of output dimensions. Therefore, a
149 number of "dummy" PTC dataset are inserted in the output list
150 that has the partially-filled PTC datasets with the covariances.
151 This output list will be used as input of
152 `PhotonTransferCurveSolveTask`, which will assemble the multiple
153 `PhotonTransferCurveDataset`s into a single one in order to fit
154 the measured covariances as a function of flux to a particular
157 Astier+19: "The Shape of the Photon Transfer Curve of CCD
158 sensors", arXiv:1905.08677.
160 ConfigClass = PhotonTransferCurveExtractConfig
161 _DefaultName =
'cpPtcExtract'
164 """Ensure that the input and output dimensions are passed along.
168 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
169 Butler to operate on.
170 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
171 Input data refs to load.
172 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
173 Output data refs to persist.
175 inputs = butlerQC.get(inputRefs)
179 inputs[
'inputDims'] = [expId.dataId[
'exposure']
for expId
in inputRefs.inputExp]
180 outputs = self.
runrun(**inputs)
181 butlerQC.put(outputs, outputRefs)
183 def run(self, inputExp, inputDims):
184 """Measure covariances from difference of flat pairs
188 inputExp : `dict` [`float`,
189 (`~lsst.afw.image.exposure.exposure.ExposureF`,
190 `~lsst.afw.image.exposure.exposure.ExposureF`, ...,
191 `~lsst.afw.image.exposure.exposure.ExposureF`)]
192 Dictionary that groups flat-field exposures that have the same
193 exposure time (seconds).
196 List of exposure IDs.
200 detector = list(inputExp.values())[0][0].getDetector()
201 detNum = detector.getId()
202 amps = detector.getAmplifiers()
203 ampNames = [amp.getName()
for amp
in amps]
204 maxMeanSignalDict = {ampName: 1e6
for ampName
in ampNames}
205 minMeanSignalDict = {ampName: 0.0
for ampName
in ampNames}
206 for ampName
in ampNames:
207 if 'ALL_AMPS' in self.config.maxMeanSignal:
208 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[
'ALL_AMPS']
209 elif ampName
in self.config.maxMeanSignal:
210 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
212 if 'ALL_AMPS' in self.config.minMeanSignal:
213 minMeanSignalDict[ampName] = self.config.minMeanSignal[
'ALL_AMPS']
214 elif ampName
in self.config.minMeanSignal:
215 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
216 tags = [(
'mu',
'<f8'), (
'afwVar',
'<f8'), (
'i',
'<i8'), (
'j',
'<i8'), (
'var',
'<f8'),
217 (
'cov',
'<f8'), (
'npix',
'<i8'), (
'ext',
'<i8'), (
'expTime',
'<f8'), (
'ampName',
'<U3')]
218 dummyPtcDataset = PhotonTransferCurveDataset(ampNames,
'DUMMY',
219 self.config.maximumRangeCovariancesAstier)
220 covArray = [np.full((self.config.maximumRangeCovariancesAstier,
221 self.config.maximumRangeCovariancesAstier), np.nan)]
222 for ampName
in ampNames:
223 dummyPtcDataset.rawExpTimes[ampName] = [np.nan]
224 dummyPtcDataset.rawMeans[ampName] = [np.nan]
225 dummyPtcDataset.rawVars[ampName] = [np.nan]
226 dummyPtcDataset.inputExpIdPairs[ampName] = [(np.nan, np.nan)]
227 dummyPtcDataset.expIdMask[ampName] = [np.nan]
228 dummyPtcDataset.covariances[ampName] = covArray
229 dummyPtcDataset.covariancesModel[ampName] = np.full_like(covArray, np.nan)
230 dummyPtcDataset.covariancesSqrtWeights[ampName] = np.full_like(covArray, np.nan)
231 dummyPtcDataset.covariancesModelNoB[ampName] = np.full_like(covArray, np.nan)
232 dummyPtcDataset.aMatrix[ampName] = np.full_like(covArray[0], np.nan)
233 dummyPtcDataset.bMatrix[ampName] = np.full_like(covArray[0], np.nan)
234 dummyPtcDataset.aMatrixNoB[ampName] = np.full_like(covArray[0], np.nan)
235 dummyPtcDataset.ptcFitPars[ampName] = [np.nan]
236 dummyPtcDataset.ptcFitParsError[ampName] = [np.nan]
237 dummyPtcDataset.ptcFitChiSq[ampName] = np.nan
238 dummyPtcDataset.finalVars[ampName] = [np.nan]
239 dummyPtcDataset.finalModelVars[ampName] = [np.nan]
240 dummyPtcDataset.finalMeans[ampName] = [np.nan]
242 partialDatasetPtcList = []
245 for i
in range(len(inputDims)):
246 partialDatasetPtcList.append(dummyPtcDataset)
248 for expTime
in inputExp:
249 exposures = inputExp[expTime]
250 if len(exposures) == 1:
251 self.log.warn(f
"Only one exposure found at expTime {expTime}. Dropping exposure "
252 f
"{exposures[0].getInfo().getVisitInfo().getExposureId()}.")
256 exp1, exp2 = exposures[0], exposures[1]
257 if len(exposures) > 2:
258 self.log.warn(f
"Already found 2 exposures at expTime {expTime}. "
259 "Ignoring exposures: "
260 f
"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}")
261 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
262 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
264 partialDatasetPtc = PhotonTransferCurveDataset(ampNames,
'',
265 self.config.maximumRangeCovariancesAstier)
266 for ampNumber, amp
in enumerate(detector):
267 ampName = amp.getName()
269 doRealSpace = self.config.covAstierRealSpace
270 if self.config.detectorMeasurementRegion ==
'AMP':
271 region = amp.getBBox()
272 elif self.config.detectorMeasurementRegion ==
'FULL':
276 muDiff, varDiff, covAstier = self.
measureMeanVarCovmeasureMeanVarCov(exp1, exp2, region=region,
277 covAstierRealSpace=doRealSpace)
279 if np.isnan(muDiff)
or np.isnan(varDiff)
or (covAstier
is None):
280 msg = (f
"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
281 f
" {expId2} of detector {detNum}.")
285 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
286 self.config.maximumRangeCovariancesAstier), np.nan)
287 covSqrtWeights = np.full_like(covArray, np.nan)
289 if (muDiff <= minMeanSignalDict[ampName])
or (muDiff >= maxMeanSignalDict[ampName]):
292 partialDatasetPtc.rawExpTimes[ampName] = [expTime]
293 partialDatasetPtc.rawMeans[ampName] = [muDiff]
294 partialDatasetPtc.rawVars[ampName] = [varDiff]
296 if covAstier
is not None:
297 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
298 ampName)
for covRow
in covAstier]
299 tempStructArray = np.array(tupleRows, dtype=tags)
301 self.config.maximumRangeCovariancesAstier)
302 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
303 partialDatasetPtc.inputExpIdPairs[ampName] = [(expId1, expId2)]
304 partialDatasetPtc.expIdMask[ampName] = [expIdMask]
305 partialDatasetPtc.covariances[ampName] = covArray
306 partialDatasetPtc.covariancesSqrtWeights[ampName] = covSqrtWeights
307 partialDatasetPtc.covariancesModel[ampName] = np.full_like(covArray, np.nan)
308 partialDatasetPtc.covariancesModelNoB[ampName] = np.full_like(covArray, np.nan)
309 partialDatasetPtc.aMatrix[ampName] = np.full_like(covArray[0], np.nan)
310 partialDatasetPtc.bMatrix[ampName] = np.full_like(covArray[0], np.nan)
311 partialDatasetPtc.aMatrixNoB[ampName] = np.full_like(covArray[0], np.nan)
312 partialDatasetPtc.ptcFitPars[ampName] = [np.nan]
313 partialDatasetPtc.ptcFitParsError[ampName] = [np.nan]
314 partialDatasetPtc.ptcFitChiSq[ampName] = np.nan
315 partialDatasetPtc.finalVars[ampName] = [np.nan]
316 partialDatasetPtc.finalModelVars[ampName] = [np.nan]
317 partialDatasetPtc.finalMeans[ampName] = [np.nan]
328 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
331 datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0]
333 datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0]
334 partialDatasetPtcList[datasetIndex] = partialDatasetPtc
335 if nAmpsNan == len(ampNames):
336 msg = f
"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
338 return pipeBase.Struct(
339 outputCovariances=partialDatasetPtcList,
343 """Calculate the mean of each of two exposures and the variance
344 and covariance of their difference. The variance is calculated
345 via afwMath, and the covariance via the methods in Astier+19
346 (appendix A). In theory, var = covariance[0,0]. This should
347 be validated, and in the future, we may decide to just keep
352 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
353 First exposure of flat field pair.
354 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
355 Second exposure of flat field pair.
356 region : `lsst.geom.Box2I`, optional
357 Region of each exposure where to perform the calculations (e.g, an amplifier).
358 covAstierRealSpace : `bool`, optional
359 Should the covariannces in Astier+19 be calculated in real space or via FFT?
360 See Appendix A of Astier+19.
364 mu : `float` or `NaN`
365 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
366 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
367 varDiff : `float` or `NaN`
368 Half of the clipped variance of the difference of the regions inthe two input
369 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
370 covDiffAstier : `list` or `NaN`
371 List with tuples of the form (dx, dy, var, cov, npix), where:
377 Variance at (dx, dy).
379 Covariance at (dx, dy).
381 Number of pixel pairs used to evaluate var and cov.
382 If either mu1 or m2 are NaN's, the returned value is NaN.
385 if region
is not None:
386 im1Area = exposure1.maskedImage[region]
387 im2Area = exposure2.maskedImage[region]
389 im1Area = exposure1.maskedImage
390 im2Area = exposure2.maskedImage
392 if self.config.binSize > 1:
393 im1Area = afwMath.binImage(im1Area, self.config.binSize)
394 im2Area = afwMath.binImage(im2Area, self.config.binSize)
396 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
397 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
398 self.config.nIterSigmaClipPtc,
400 im1StatsCtrl.setNanSafe(
True)
401 im1StatsCtrl.setAndMask(im1MaskVal)
403 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
404 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
405 self.config.nIterSigmaClipPtc,
407 im2StatsCtrl.setNanSafe(
True)
408 im2StatsCtrl.setAndMask(im2MaskVal)
411 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
412 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
413 if np.isnan(mu1)
or np.isnan(mu2):
414 self.log.warn(f
"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
415 return np.nan, np.nan,
None
420 temp = im2Area.clone()
422 diffIm = im1Area.clone()
427 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
428 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
429 self.config.nIterSigmaClipPtc,
431 diffImStatsCtrl.setNanSafe(
True)
432 diffImStatsCtrl.setAndMask(diffImMaskVal)
434 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
437 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
438 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
441 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
444 if np.sum(w) < self.config.minNumberGoodPixelsForFft:
445 self.log.warn(f
"Number of good points for FFT ({np.sum(w)}) is less than threshold "
446 f
"({self.config.minNumberGoodPixelsForFft})")
447 return np.nan, np.nan,
None
449 maxRangeCov = self.config.maximumRangeCovariancesAstier
450 if covAstierRealSpace:
453 shapeDiff = diffIm.image.array.shape
454 fftShape = (
fftSize(shapeDiff[0] + maxRangeCov),
fftSize(shapeDiff[1]+maxRangeCov))
455 c =
CovFft(diffIm.image.array, w, fftShape, maxRangeCov)
456 covDiffAstier = c.reportCovFft(maxRangeCov)
458 return mu, varDiff, covDiffAstier
def makeCovArray(inputTuple, maxRangeFromTuple=8)
def computeCovDirect(diffImage, weightImage, maxRange)
def arrangeFlatsByExpTime(exposureList)