Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.afw.math as afwMath
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
28 sigmaClipCorrection)
30import lsst.pipe.base.connectionTypes as cT
32from .astierCovPtcUtils import (CovFastFourierTransform, computeCovDirect)
33from .astierCovPtcFit import makeCovArray
35from lsst.ip.isr import PhotonTransferCurveDataset
36from lsst.ip.isr import IsrTask
38__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
41class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "detector")):
44 inputExp = cT.Input(
45 name="ptcInputExposurePairs",
46 doc="Input post-ISR processed exposure pairs (flats) to"
47 "measure covariances from.",
48 storageClass="Exposure",
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 deferLoad=False,
52 )
54 outputCovariances = cT.Output(
55 name="ptcCovariances",
56 doc="Extracted flat (co)variances.",
57 storageClass="PhotonTransferCurveDataset",
58 dimensions=("instrument", "exposure", "detector"),
59 multiple=True,
60 )
63class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
64 pipelineConnections=PhotonTransferCurveExtractConnections):
65 """Configuration for the measurement of covariances from flats.
66 """
67 matchByExposureId = pexConfig.Field(
68 dtype=bool,
69 doc="Should exposures be matched by ID rather than exposure time?",
70 default=False,
71 )
72 maximumRangeCovariancesAstier = pexConfig.Field(
73 dtype=int,
74 doc="Maximum range of covariances as in Astier+19",
75 default=8,
76 )
77 covAstierRealSpace = pexConfig.Field(
78 dtype=bool,
79 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
80 default=False,
81 )
82 binSize = pexConfig.Field(
83 dtype=int,
84 doc="Bin the image by this factor in both dimensions.",
85 default=1,
86 )
87 minMeanSignal = pexConfig.DictField(
88 keytype=str,
89 itemtype=float,
90 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
91 " The same cut is applied to all amps if this dictionary is of the form"
92 " {'ALL_AMPS': value}",
93 default={'ALL_AMPS': 0.0},
94 )
95 maxMeanSignal = pexConfig.DictField(
96 keytype=str,
97 itemtype=float,
98 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
99 " The same cut is applied to all amps if this dictionary is of the form"
100 " {'ALL_AMPS': value}",
101 default={'ALL_AMPS': 1e6},
102 )
103 maskNameList = pexConfig.ListField(
104 dtype=str,
105 doc="Mask list to exclude from statistics calculations.",
106 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'],
107 )
108 nSigmaClipPtc = pexConfig.Field(
109 dtype=float,
110 doc="Sigma cut for afwMath.StatisticsControl()",
111 default=5.5,
112 )
113 nIterSigmaClipPtc = pexConfig.Field(
114 dtype=int,
115 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
116 default=3,
117 )
118 minNumberGoodPixelsForCovariance = pexConfig.Field(
119 dtype=int,
120 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
121 " direclty).",
122 default=10000,
123 )
124 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
125 dtype=float,
126 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
127 "for a region of a difference image is greater than this threshold (percentage), "
128 "a warning will be issued.",
129 default=1.,
130 )
131 detectorMeasurementRegion = pexConfig.ChoiceField(
132 dtype=str,
133 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
134 default='AMP',
135 allowed={
136 "AMP": "Amplifier of the detector.",
137 "FULL": "Full image."
138 }
139 )
140 numEdgeSuspect = pexConfig.Field(
141 dtype=int,
142 doc="Number of edge pixels to be flagged as untrustworthy.",
143 default=0,
144 )
145 edgeMaskLevel = pexConfig.ChoiceField(
146 dtype=str,
147 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
148 default="DETECTOR",
149 allowed={
150 'DETECTOR': 'Mask only the edges of the full detector.',
151 'AMP': 'Mask edges of each amplifier.',
152 },
153 )
156class PhotonTransferCurveExtractTask(pipeBase.PipelineTask,
157 pipeBase.CmdLineTask):
158 """Task to measure covariances from flat fields.
159 This task receives as input a list of flat-field images
160 (flats), and sorts these flats in pairs taken at the
161 same time (if there's a different number of flats,
162 those flats are discarded). The mean, variance, and
163 covariances are measured from the difference of the flat
164 pairs at a given time. The variance is calculated
165 via afwMath, and the covariance via the methods in Astier+19
166 (appendix A). In theory, var = covariance[0,0]. This should
167 be validated, and in the future, we may decide to just keep
168 one (covariance).
170 The measured covariances at a particular time (along with
171 other quantities such as the mean) are stored in a PTC dataset
172 object (`PhotonTransferCurveDataset`), which gets partially
173 filled. The number of partially-filled PTC dataset objects
174 will be less than the number of input exposures, but gen3
175 requires/assumes that the number of input dimensions matches
176 bijectively the number of output dimensions. Therefore, a
177 number of "dummy" PTC dataset are inserted in the output list
178 that has the partially-filled PTC datasets with the covariances.
179 This output list will be used as input of
180 `PhotonTransferCurveSolveTask`, which will assemble the multiple
181 `PhotonTransferCurveDataset`s into a single one in order to fit
182 the measured covariances as a function of flux to a particular
183 model.
185 Astier+19: "The Shape of the Photon Transfer Curve of CCD
186 sensors", arXiv:1905.08677.
187 """
188 ConfigClass = PhotonTransferCurveExtractConfig
189 _DefaultName = 'cpPtcExtract'
191 def runQuantum(self, butlerQC, inputRefs, outputRefs):
192 """Ensure that the input and output dimensions are passed along.
194 Parameters
195 ----------
196 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
197 Butler to operate on.
198 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
199 Input data refs to load.
200 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
201 Output data refs to persist.
202 """
203 inputs = butlerQC.get(inputRefs)
204 # Ids of input list of exposures
205 inputs['inputDims'] = [expId.dataId['exposure'] for expId in inputRefs.inputExp]
207 # Dictionary, keyed by expTime, with tuples containing flat exposures and their IDs.
208 if self.config.matchByExposureId:
209 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims'])
210 else:
211 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims'])
213 outputs = self.run(**inputs)
214 butlerQC.put(outputs, outputRefs)
216 def run(self, inputExp, inputDims):
217 """Measure covariances from difference of flat pairs
219 Parameters
220 ----------
221 inputExp : `dict` [`float`,
222 (`~lsst.afw.image.exposure.exposure.ExposureF`,
223 `~lsst.afw.image.exposure.exposure.ExposureF`, ...,
224 `~lsst.afw.image.exposure.exposure.ExposureF`)]
225 Dictionary that groups flat-field exposures that have the same
226 exposure time (seconds).
228 inputDims : `list`
229 List of exposure IDs.
230 """
231 # inputExp.values() returns a view, which we turn into a list. We then
232 # access the first exposure-ID tuple to get the detector.
233 detector = list(inputExp.values())[0][0][0].getDetector()
234 detNum = detector.getId()
235 amps = detector.getAmplifiers()
236 ampNames = [amp.getName() for amp in amps]
238 # Each amp may have a different min and max ADU signal specified in the config.
239 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
240 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
241 for ampName in ampNames:
242 if 'ALL_AMPS' in self.config.maxMeanSignal:
243 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
244 elif ampName in self.config.maxMeanSignal:
245 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
247 if 'ALL_AMPS' in self.config.minMeanSignal:
248 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
249 elif ampName in self.config.minMeanSignal:
250 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
251 # These are the column names for `tupleRows` below.
252 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
253 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
254 # Create a dummy ptcDataset
255 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
256 self.config.maximumRangeCovariancesAstier)
257 # Initialize amps of `dummyPtcDatset`.
258 for ampName in ampNames:
259 dummyPtcDataset.setAmpValues(ampName)
260 # Output list with PTC datasets.
261 partialPtcDatasetList = []
262 # The number of output references needs to match that of input references:
263 # initialize outputlist with dummy PTC datasets.
264 for i in range(len(inputDims)):
265 partialPtcDatasetList.append(dummyPtcDataset)
267 if self.config.numEdgeSuspect > 0:
268 isrTask = IsrTask()
269 self.log.info(f"Masking {self.config.numEdgeSuspect} pixels from the edges "
270 "of all exposures as SUSPECT.")
272 for expTime in inputExp:
273 exposures = inputExp[expTime]
274 if len(exposures) == 1:
275 self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure "
276 f"{exposures[0][1]}")
277 continue
278 else:
279 # Only use the first two exposures at expTime. Each elements is a tuple (exposure, expId)
280 exp1, expId1 = exposures[0]
281 exp2, expId2 = exposures[1]
282 if len(exposures) > 2:
283 self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
284 "Ignoring exposures: "
285 f"{i[1] for i in exposures[2:]}")
286 # Mask pixels at the edge of the detector or of each amp
287 if self.config.numEdgeSuspect > 0:
288 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
289 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
290 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
291 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
293 nAmpsNan = 0
294 partialPtcDataset = PhotonTransferCurveDataset(ampNames, '',
295 self.config.maximumRangeCovariancesAstier)
296 for ampNumber, amp in enumerate(detector):
297 ampName = amp.getName()
298 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for (i,j) in {maxLag, maxLag}^2]
299 doRealSpace = self.config.covAstierRealSpace
300 if self.config.detectorMeasurementRegion == 'AMP':
301 region = amp.getBBox()
302 elif self.config.detectorMeasurementRegion == 'FULL':
303 region = None
304 # `measureMeanVarCov` is the function that measures the variance and covariances from
305 # the difference image of two flats at the same exposure time.
306 # The variable `covAstier` is of the form: [(i, j, var (cov[0,0]), cov, npix) for (i,j)
307 # in {maxLag, maxLag}^2]
308 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region,
309 covAstierRealSpace=doRealSpace)
310 # Correction factor for sigma clipping. Function returns 1/sqrt(varFactor),
311 # so it needs to be squared. varDiff is calculated via afwMath.VARIANCECLIP.
312 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2
313 varDiff *= varFactor
315 expIdMask = True
316 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
317 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
318 f" {expId2} of detector {detNum}.")
319 self.log.warn(msg)
320 nAmpsNan += 1
321 expIdMask = False
322 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
323 self.config.maximumRangeCovariancesAstier), np.nan)
324 covSqrtWeights = np.full_like(covArray, np.nan)
326 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
327 expIdMask = False
329 if covAstier is not None:
330 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
331 ampName) for covRow in covAstier]
332 tempStructArray = np.array(tupleRows, dtype=tags)
333 covArray, vcov, _ = makeCovArray(tempStructArray,
334 self.config.maximumRangeCovariancesAstier)
335 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
337 # Correct covArray for sigma clipping:
338 # 1) Apply varFactor twice for the whole covariance matrix
339 covArray *= varFactor**2
340 # 2) But, only once for the variance element of the matrix, covArray[0,0]
341 covArray[0, 0] /= varFactor
343 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff],
344 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)],
345 expIdMask=[expIdMask], covArray=covArray,
346 covSqrtWeights=covSqrtWeights)
347 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
348 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple
349 # with a single-element array, so [0][0]
350 # is necessary to extract the required index.
351 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
352 partialPtcDatasetList[datasetIndex] = partialPtcDataset
354 if nAmpsNan == len(ampNames):
355 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
356 self.log.warn(msg)
357 return pipeBase.Struct(
358 outputCovariances=partialPtcDatasetList,
359 )
361 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
362 """Calculate the mean of each of two exposures and the variance
363 and covariance of their difference. The variance is calculated
364 via afwMath, and the covariance via the methods in Astier+19
365 (appendix A). In theory, var = covariance[0,0]. This should
366 be validated, and in the future, we may decide to just keep
367 one (covariance).
369 Parameters
370 ----------
371 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
372 First exposure of flat field pair.
373 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
374 Second exposure of flat field pair.
375 region : `lsst.geom.Box2I`, optional
376 Region of each exposure where to perform the calculations (e.g, an amplifier).
377 covAstierRealSpace : `bool`, optional
378 Should the covariannces in Astier+19 be calculated in real space or via FFT?
379 See Appendix A of Astier+19.
381 Returns
382 -------
383 mu : `float` or `NaN`
384 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
385 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
386 varDiff : `float` or `NaN`
387 Half of the clipped variance of the difference of the regions inthe two input
388 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
389 covDiffAstier : `list` or `NaN`
390 List with tuples of the form (dx, dy, var, cov, npix), where:
391 dx : `int`
392 Lag in x
393 dy : `int`
394 Lag in y
395 var : `float`
396 Variance at (dx, dy).
397 cov : `float`
398 Covariance at (dx, dy).
399 nPix : `int`
400 Number of pixel pairs used to evaluate var and cov.
401 If either mu1 or m2 are NaN's, the returned value is NaN.
402 """
404 if region is not None:
405 im1Area = exposure1.maskedImage[region]
406 im2Area = exposure2.maskedImage[region]
407 else:
408 im1Area = exposure1.maskedImage
409 im2Area = exposure2.maskedImage
411 if self.config.binSize > 1:
412 im1Area = afwMath.binImage(im1Area, self.config.binSize)
413 im2Area = afwMath.binImage(im2Area, self.config.binSize)
415 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
416 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
417 self.config.nIterSigmaClipPtc,
418 im1MaskVal)
419 im1StatsCtrl.setNanSafe(True)
420 im1StatsCtrl.setAndMask(im1MaskVal)
422 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
423 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
424 self.config.nIterSigmaClipPtc,
425 im2MaskVal)
426 im2StatsCtrl.setNanSafe(True)
427 im2StatsCtrl.setAndMask(im2MaskVal)
429 # Clipped mean of images; then average of mean.
430 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
431 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
432 if np.isnan(mu1) or np.isnan(mu2):
433 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
434 return np.nan, np.nan, None
435 mu = 0.5*(mu1 + mu2)
437 # Take difference of pairs
438 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
439 temp = im2Area.clone()
440 temp *= mu1
441 diffIm = im1Area.clone()
442 diffIm *= mu2
443 diffIm -= temp
444 diffIm /= mu
446 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
447 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
448 self.config.nIterSigmaClipPtc,
449 diffImMaskVal)
450 diffImStatsCtrl.setNanSafe(True)
451 diffImStatsCtrl.setAndMask(diffImMaskVal)
453 # Variance calculation via afwMath
454 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
456 # Covariances calculations
457 # Get the pixels that were not clipped
458 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()
459 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, diffImStatsCtrl).getValue()
460 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
461 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
463 # Get the pixels in the mask planes of teh differenc eimage that were ignored
464 # by the clipping algorithm
465 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
466 # Combine the two sets of pixels ('1': use; '0': don't use) into a final weight matrix
467 # to be used in the covariance calculations below.
468 w = unmasked*wDiff
470 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance:
471 self.log.warn(f"Number of good points for covariance calculation ({np.sum(w)}) is less "
472 f"(than threshold {self.config.minNumberGoodPixelsForCovariance})")
473 return np.nan, np.nan, None
475 maxRangeCov = self.config.maximumRangeCovariancesAstier
476 if covAstierRealSpace:
477 # Calculate covariances in real space.
478 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov)
479 else:
480 # Calculate covariances via FFT (default).
481 shapeDiff = np.array(diffIm.image.array.shape)
482 # Calculate the sizes of FFT dimensions.
483 s = shapeDiff + maxRangeCov
484 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
485 fftSize = np.array(2**(tempSize+1)).astype(int)
486 fftShape = (fftSize[0], fftSize[1])
488 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
489 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
491 # Compare Cov[0,0] and afwMath.VARIANCECLIP
492 # covDiffAstier[0] is the Cov[0,0] element, [3] is the variance, and there's a factor of 0.5
493 # difference with afwMath.VARIANCECLIP.
494 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
495 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
496 if fractionalDiff >= thresholdPercentage:
497 self.log.warn("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
498 f"is more than {thresholdPercentage}%: {fractionalDiff}")
500 return mu, varDiff, covDiffAstier