Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py: 15%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.afw.math as afwMath
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
28 sigmaClipCorrection)
30import lsst.pipe.base.connectionTypes as cT
32from .astierCovPtcUtils import (CovFastFourierTransform, computeCovDirect)
33from .astierCovPtcFit import makeCovArray
35from lsst.ip.isr import PhotonTransferCurveDataset
36from lsst.ip.isr import IsrTask
38__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
41class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "detector")):
44 inputExp = cT.Input(
45 name="ptcInputExposurePairs",
46 doc="Input post-ISR processed exposure pairs (flats) to"
47 "measure covariances from.",
48 storageClass="Exposure",
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 deferLoad=False,
52 )
54 outputCovariances = cT.Output(
55 name="ptcCovariances",
56 doc="Extracted flat (co)variances.",
57 storageClass="PhotonTransferCurveDataset",
58 dimensions=("instrument", "exposure", "detector"),
59 multiple=True,
60 )
63class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
64 pipelineConnections=PhotonTransferCurveExtractConnections):
65 """Configuration for the measurement of covariances from flats.
66 """
68 matchByExposureId = pexConfig.Field(
69 dtype=bool,
70 doc="Should exposures be matched by ID rather than exposure time?",
71 default=False,
72 )
73 maximumRangeCovariancesAstier = pexConfig.Field(
74 dtype=int,
75 doc="Maximum range of covariances as in Astier+19",
76 default=8,
77 )
78 covAstierRealSpace = pexConfig.Field(
79 dtype=bool,
80 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
81 default=False,
82 )
83 binSize = pexConfig.Field(
84 dtype=int,
85 doc="Bin the image by this factor in both dimensions.",
86 default=1,
87 )
88 minMeanSignal = pexConfig.DictField(
89 keytype=str,
90 itemtype=float,
91 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
92 " The same cut is applied to all amps if this dictionary is of the form"
93 " {'ALL_AMPS': value}",
94 default={'ALL_AMPS': 0.0},
95 )
96 maxMeanSignal = pexConfig.DictField(
97 keytype=str,
98 itemtype=float,
99 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
100 " The same cut is applied to all amps if this dictionary is of the form"
101 " {'ALL_AMPS': value}",
102 default={'ALL_AMPS': 1e6},
103 )
104 maskNameList = pexConfig.ListField(
105 dtype=str,
106 doc="Mask list to exclude from statistics calculations.",
107 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'],
108 )
109 nSigmaClipPtc = pexConfig.Field(
110 dtype=float,
111 doc="Sigma cut for afwMath.StatisticsControl()",
112 default=5.5,
113 )
114 nIterSigmaClipPtc = pexConfig.Field(
115 dtype=int,
116 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
117 default=3,
118 )
119 minNumberGoodPixelsForCovariance = pexConfig.Field(
120 dtype=int,
121 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
122 " direclty).",
123 default=10000,
124 )
125 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
126 dtype=float,
127 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
128 "for a region of a difference image is greater than this threshold (percentage), "
129 "a warning will be issued.",
130 default=1.,
131 )
132 detectorMeasurementRegion = pexConfig.ChoiceField(
133 dtype=str,
134 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
135 default='AMP',
136 allowed={
137 "AMP": "Amplifier of the detector.",
138 "FULL": "Full image."
139 }
140 )
141 numEdgeSuspect = pexConfig.Field(
142 dtype=int,
143 doc="Number of edge pixels to be flagged as untrustworthy.",
144 default=0,
145 )
146 edgeMaskLevel = pexConfig.ChoiceField(
147 dtype=str,
148 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
149 default="DETECTOR",
150 allowed={
151 'DETECTOR': 'Mask only the edges of the full detector.',
152 'AMP': 'Mask edges of each amplifier.',
153 },
154 )
157class PhotonTransferCurveExtractTask(pipeBase.PipelineTask,
158 pipeBase.CmdLineTask):
159 """Task to measure covariances from flat fields.
161 This task receives as input a list of flat-field images
162 (flats), and sorts these flats in pairs taken at the
163 same time (if there's a different number of flats,
164 those flats are discarded). The mean, variance, and
165 covariances are measured from the difference of the flat
166 pairs at a given time. The variance is calculated
167 via afwMath, and the covariance via the methods in Astier+19
168 (appendix A). In theory, var = covariance[0,0]. This should
169 be validated, and in the future, we may decide to just keep
170 one (covariance).
172 The measured covariances at a particular time (along with other
173 quantities such as the mean) are stored in a PTC dataset object
174 (`~lsst.ip.isr.PhotonTransferCurveDataset`), which gets
175 partially filled. The number of partially-filled PTC dataset
176 objects will be less than the number of input exposures, but gen3
177 requires/assumes that the number of input dimensions matches
178 bijectively the number of output dimensions. Therefore, a number
179 of "dummy" PTC dataset are inserted in the output list that has
180 the partially-filled PTC datasets with the covariances. This
181 output list will be used as input of
182 ``PhotonTransferCurveSolveTask``, which will assemble the multiple
183 ``PhotonTransferCurveDataset`` into a single one in order to fit
184 the measured covariances as a function of flux to a particular
185 model.
187 Astier+19: "The Shape of the Photon Transfer Curve of CCD
188 sensors", arXiv:1905.08677.
189 """
191 ConfigClass = PhotonTransferCurveExtractConfig
192 _DefaultName = 'cpPtcExtract'
194 def runQuantum(self, butlerQC, inputRefs, outputRefs):
195 """Ensure that the input and output dimensions are passed along.
197 Parameters
198 ----------
199 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
200 Butler to operate on.
201 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
202 Input data refs to load.
203 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
204 Output data refs to persist.
205 """
206 inputs = butlerQC.get(inputRefs)
207 # Ids of input list of exposures
208 inputs['inputDims'] = [expId.dataId['exposure'] for expId in inputRefs.inputExp]
210 # Dictionary, keyed by expTime, with tuples containing flat
211 # exposures and their IDs.
212 if self.config.matchByExposureId:
213 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims'])
214 else:
215 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims'])
217 outputs = self.run(**inputs)
218 butlerQC.put(outputs, outputRefs)
220 def run(self, inputExp, inputDims):
221 """Measure covariances from difference of flat pairs
223 Parameters
224 ----------
225 inputExp : `dict` [`float`, `list` [`~lsst.afw.image.ExposureF`]]
226 Dictionary that groups flat-field exposures that have the same
227 exposure time (seconds).
229 inputDims : `list`
230 List of exposure IDs.
232 Returns
233 -------
234 results : `lsst.pipe.base.Struct`
235 The results struct containing:
237 ``outputCovariances``
238 A list containing the per-pair PTC measurements (`list`
239 [`lsst.ip.isr.PhotonTransferCurveDataset`])
240 """
241 # inputExp.values() returns a view, which we turn into a list. We then
242 # access the first exposure-ID tuple to get the detector.
243 detector = list(inputExp.values())[0][0][0].getDetector()
244 detNum = detector.getId()
245 amps = detector.getAmplifiers()
246 ampNames = [amp.getName() for amp in amps]
248 # Each amp may have a different min and max ADU signal
249 # specified in the config.
250 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
251 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
252 for ampName in ampNames:
253 if 'ALL_AMPS' in self.config.maxMeanSignal:
254 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
255 elif ampName in self.config.maxMeanSignal:
256 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
258 if 'ALL_AMPS' in self.config.minMeanSignal:
259 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
260 elif ampName in self.config.minMeanSignal:
261 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
262 # These are the column names for `tupleRows` below.
263 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
264 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
265 # Create a dummy ptcDataset
266 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
267 self.config.maximumRangeCovariancesAstier)
268 # Initialize amps of `dummyPtcDatset`.
269 for ampName in ampNames:
270 dummyPtcDataset.setAmpValues(ampName)
271 # Output list with PTC datasets.
272 partialPtcDatasetList = []
273 # The number of output references needs to match that of input
274 # references: initialize outputlist with dummy PTC datasets.
275 for i in range(len(inputDims)):
276 partialPtcDatasetList.append(dummyPtcDataset)
278 if self.config.numEdgeSuspect > 0:
279 isrTask = IsrTask()
280 self.log.info(f"Masking {self.config.numEdgeSuspect} pixels from the edges "
281 "of all exposures as SUSPECT.")
283 for expTime in inputExp:
284 exposures = inputExp[expTime]
285 if len(exposures) == 1:
286 self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure "
287 f"{exposures[0][1]}")
288 continue
289 else:
290 # Only use the first two exposures at expTime. Each
291 # elements is a tuple (exposure, expId)
292 exp1, expId1 = exposures[0]
293 exp2, expId2 = exposures[1]
294 if len(exposures) > 2:
295 self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
296 "Ignoring exposures: "
297 f"{i[1] for i in exposures[2:]}")
298 # Mask pixels at the edge of the detector or of each amp
299 if self.config.numEdgeSuspect > 0:
300 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
301 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
302 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
303 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
305 nAmpsNan = 0
306 partialPtcDataset = PhotonTransferCurveDataset(ampNames, 'PARTIAL',
307 self.config.maximumRangeCovariancesAstier)
308 for ampNumber, amp in enumerate(detector):
309 ampName = amp.getName()
310 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for
311 # (i,j) in {maxLag, maxLag}^2]
312 doRealSpace = self.config.covAstierRealSpace
313 if self.config.detectorMeasurementRegion == 'AMP':
314 region = amp.getBBox()
315 elif self.config.detectorMeasurementRegion == 'FULL':
316 region = None
317 # `measureMeanVarCov` is the function that measures
318 # the variance and covariances from the difference
319 # image of two flats at the same exposure time. The
320 # variable `covAstier` is of the form: [(i, j, var
321 # (cov[0,0]), cov, npix) for (i,j) in {maxLag,
322 # maxLag}^2]
323 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region,
324 covAstierRealSpace=doRealSpace)
325 # Correction factor for sigma clipping. Function
326 # returns 1/sqrt(varFactor), so it needs to be
327 # squared. varDiff is calculated via
328 # afwMath.VARIANCECLIP.
329 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2
330 varDiff *= varFactor
332 expIdMask = True
333 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
334 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
335 f" {expId2} of detector {detNum}.")
336 self.log.warn(msg)
337 nAmpsNan += 1
338 expIdMask = False
339 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
340 self.config.maximumRangeCovariancesAstier), np.nan)
341 covSqrtWeights = np.full_like(covArray, np.nan)
343 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
344 expIdMask = False
346 if covAstier is not None:
347 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
348 ampName) for covRow in covAstier]
349 tempStructArray = np.array(tupleRows, dtype=tags)
350 covArray, vcov, _ = makeCovArray(tempStructArray,
351 self.config.maximumRangeCovariancesAstier)
352 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
354 # Correct covArray for sigma clipping:
355 # 1) Apply varFactor twice for the whole covariance matrix
356 covArray *= varFactor**2
357 # 2) But, only once for the variance element of the
358 # matrix, covArray[0,0]
359 covArray[0, 0] /= varFactor
361 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff],
362 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)],
363 expIdMask=[expIdMask], covArray=covArray,
364 covSqrtWeights=covSqrtWeights)
365 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
366 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple
367 # with a single-element array, so [0][0]
368 # is necessary to extract the required index.
369 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
370 partialPtcDatasetList[datasetIndex] = partialPtcDataset
372 if nAmpsNan == len(ampNames):
373 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
374 self.log.warn(msg)
375 return pipeBase.Struct(
376 outputCovariances=partialPtcDatasetList,
377 )
379 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
380 """Calculate the mean of each of two exposures and the variance
381 and covariance of their difference. The variance is calculated
382 via afwMath, and the covariance via the methods in Astier+19
383 (appendix A). In theory, var = covariance[0,0]. This should
384 be validated, and in the future, we may decide to just keep
385 one (covariance).
387 Parameters
388 ----------
389 exposure1 : `lsst.afw.image.exposure.ExposureF`
390 First exposure of flat field pair.
391 exposure2 : `lsst.afw.image.exposure.ExposureF`
392 Second exposure of flat field pair.
393 region : `lsst.geom.Box2I`, optional
394 Region of each exposure where to perform the calculations
395 (e.g, an amplifier).
396 covAstierRealSpace : `bool`, optional
397 Should the covariannces in Astier+19 be calculated in real
398 space or via FFT? See Appendix A of Astier+19.
400 Returns
401 -------
402 mu : `float` or `NaN`
403 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means
404 of the regions in both exposures. If either mu1 or m2 are
405 NaN's, the returned value is NaN.
406 varDiff : `float` or `NaN`
407 Half of the clipped variance of the difference of the
408 regions inthe two input exposures. If either mu1 or m2 are
409 NaN's, the returned value is NaN.
410 covDiffAstier : `list` or `NaN`
411 List with tuples of the form (dx, dy, var, cov, npix), where:
412 dx : `int`
413 Lag in x
414 dy : `int`
415 Lag in y
416 var : `float`
417 Variance at (dx, dy).
418 cov : `float`
419 Covariance at (dx, dy).
420 nPix : `int`
421 Number of pixel pairs used to evaluate var and cov.
423 If either mu1 or m2 are NaN's, the returned value is NaN.
424 """
425 if region is not None:
426 im1Area = exposure1.maskedImage[region]
427 im2Area = exposure2.maskedImage[region]
428 else:
429 im1Area = exposure1.maskedImage
430 im2Area = exposure2.maskedImage
432 if self.config.binSize > 1:
433 im1Area = afwMath.binImage(im1Area, self.config.binSize)
434 im2Area = afwMath.binImage(im2Area, self.config.binSize)
436 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
437 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
438 self.config.nIterSigmaClipPtc,
439 im1MaskVal)
440 im1StatsCtrl.setNanSafe(True)
441 im1StatsCtrl.setAndMask(im1MaskVal)
443 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
444 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
445 self.config.nIterSigmaClipPtc,
446 im2MaskVal)
447 im2StatsCtrl.setNanSafe(True)
448 im2StatsCtrl.setAndMask(im2MaskVal)
450 # Clipped mean of images; then average of mean.
451 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
452 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
453 if np.isnan(mu1) or np.isnan(mu2):
454 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
455 return np.nan, np.nan, None
456 mu = 0.5*(mu1 + mu2)
458 # Take difference of pairs
459 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
460 temp = im2Area.clone()
461 temp *= mu1
462 diffIm = im1Area.clone()
463 diffIm *= mu2
464 diffIm -= temp
465 diffIm /= mu
467 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
468 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
469 self.config.nIterSigmaClipPtc,
470 diffImMaskVal)
471 diffImStatsCtrl.setNanSafe(True)
472 diffImStatsCtrl.setAndMask(diffImMaskVal)
474 # Variance calculation via afwMath
475 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
477 # Covariances calculations
478 # Get the pixels that were not clipped
479 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()
480 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, diffImStatsCtrl).getValue()
481 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
482 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
484 # Get the pixels in the mask planes of the difference image
485 # that were ignored by the clipping algorithm
486 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
487 # Combine the two sets of pixels ('1': use; '0': don't use)
488 # into a final weight matrix to be used in the covariance
489 # calculations below.
490 w = unmasked*wDiff
492 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance:
493 self.log.warn(f"Number of good points for covariance calculation ({np.sum(w)}) is less "
494 f"(than threshold {self.config.minNumberGoodPixelsForCovariance})")
495 return np.nan, np.nan, None
497 maxRangeCov = self.config.maximumRangeCovariancesAstier
498 if covAstierRealSpace:
499 # Calculate covariances in real space.
500 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov)
501 else:
502 # Calculate covariances via FFT (default).
503 shapeDiff = np.array(diffIm.image.array.shape)
504 # Calculate the sizes of FFT dimensions.
505 s = shapeDiff + maxRangeCov
506 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
507 fftSize = np.array(2**(tempSize+1)).astype(int)
508 fftShape = (fftSize[0], fftSize[1])
510 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
511 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
513 # Compare Cov[0,0] and afwMath.VARIANCECLIP covDiffAstier[0]
514 # is the Cov[0,0] element, [3] is the variance, and there's a
515 # factor of 0.5 difference with afwMath.VARIANCECLIP.
516 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
517 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
518 if fractionalDiff >= thresholdPercentage:
519 self.log.warn("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
520 f"is more than {thresholdPercentage}%: {fractionalDiff}")
522 return mu, varDiff, covDiffAstier