Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.afw.math as afwMath
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
28 sigmaClipCorrection)
30import lsst.pipe.base.connectionTypes as cT
32from .astierCovPtcUtils import (CovFastFourierTransform, computeCovDirect)
33from .astierCovPtcFit import makeCovArray
35from lsst.ip.isr import PhotonTransferCurveDataset
36from lsst.ip.isr import IsrTask
38__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
41class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "detector")):
44 inputExp = cT.Input(
45 name="ptcInputExposurePairs",
46 doc="Input post-ISR processed exposure pairs (flats) to"
47 "measure covariances from.",
48 storageClass="Exposure",
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 deferLoad=True,
52 )
54 outputCovariances = cT.Output(
55 name="ptcCovariances",
56 doc="Extracted flat (co)variances.",
57 storageClass="PhotonTransferCurveDataset",
58 dimensions=("instrument", "exposure", "detector"),
59 multiple=True,
60 )
63class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
64 pipelineConnections=PhotonTransferCurveExtractConnections):
65 """Configuration for the measurement of covariances from flats.
66 """
68 matchByExposureId = pexConfig.Field(
69 dtype=bool,
70 doc="Should exposures be matched by ID rather than exposure time?",
71 default=False,
72 )
73 maximumRangeCovariancesAstier = pexConfig.Field(
74 dtype=int,
75 doc="Maximum range of covariances as in Astier+19",
76 default=8,
77 )
78 covAstierRealSpace = pexConfig.Field(
79 dtype=bool,
80 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
81 default=False,
82 )
83 binSize = pexConfig.Field(
84 dtype=int,
85 doc="Bin the image by this factor in both dimensions.",
86 default=1,
87 )
88 minMeanSignal = pexConfig.DictField(
89 keytype=str,
90 itemtype=float,
91 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
92 " The same cut is applied to all amps if this dictionary is of the form"
93 " {'ALL_AMPS': value}",
94 default={'ALL_AMPS': 0.0},
95 )
96 maxMeanSignal = pexConfig.DictField(
97 keytype=str,
98 itemtype=float,
99 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
100 " The same cut is applied to all amps if this dictionary is of the form"
101 " {'ALL_AMPS': value}",
102 default={'ALL_AMPS': 1e6},
103 )
104 maskNameList = pexConfig.ListField(
105 dtype=str,
106 doc="Mask list to exclude from statistics calculations.",
107 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'],
108 )
109 nSigmaClipPtc = pexConfig.Field(
110 dtype=float,
111 doc="Sigma cut for afwMath.StatisticsControl()",
112 default=5.5,
113 )
114 nIterSigmaClipPtc = pexConfig.Field(
115 dtype=int,
116 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
117 default=3,
118 )
119 minNumberGoodPixelsForCovariance = pexConfig.Field(
120 dtype=int,
121 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
122 " direclty).",
123 default=10000,
124 )
125 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
126 dtype=float,
127 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
128 "for a region of a difference image is greater than this threshold (percentage), "
129 "a warning will be issued.",
130 default=1.,
131 )
132 detectorMeasurementRegion = pexConfig.ChoiceField(
133 dtype=str,
134 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
135 default='AMP',
136 allowed={
137 "AMP": "Amplifier of the detector.",
138 "FULL": "Full image."
139 }
140 )
141 numEdgeSuspect = pexConfig.Field(
142 dtype=int,
143 doc="Number of edge pixels to be flagged as untrustworthy.",
144 default=0,
145 )
146 edgeMaskLevel = pexConfig.ChoiceField(
147 dtype=str,
148 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
149 default="DETECTOR",
150 allowed={
151 'DETECTOR': 'Mask only the edges of the full detector.',
152 'AMP': 'Mask edges of each amplifier.',
153 },
154 )
157class PhotonTransferCurveExtractTask(pipeBase.PipelineTask,
158 pipeBase.CmdLineTask):
159 """Task to measure covariances from flat fields.
161 This task receives as input a list of flat-field images
162 (flats), and sorts these flats in pairs taken at the
163 same time (if there's a different number of flats,
164 those flats are discarded). The mean, variance, and
165 covariances are measured from the difference of the flat
166 pairs at a given time. The variance is calculated
167 via afwMath, and the covariance via the methods in Astier+19
168 (appendix A). In theory, var = covariance[0,0]. This should
169 be validated, and in the future, we may decide to just keep
170 one (covariance).
172 The measured covariances at a particular time (along with other
173 quantities such as the mean) are stored in a PTC dataset object
174 (`~lsst.ip.isr.PhotonTransferCurveDataset`), which gets
175 partially filled. The number of partially-filled PTC dataset
176 objects will be less than the number of input exposures, but gen3
177 requires/assumes that the number of input dimensions matches
178 bijectively the number of output dimensions. Therefore, a number
179 of "dummy" PTC dataset are inserted in the output list that has
180 the partially-filled PTC datasets with the covariances. This
181 output list will be used as input of
182 ``PhotonTransferCurveSolveTask``, which will assemble the multiple
183 ``PhotonTransferCurveDataset`` into a single one in order to fit
184 the measured covariances as a function of flux to a particular
185 model.
187 Astier+19: "The Shape of the Photon Transfer Curve of CCD
188 sensors", arXiv:1905.08677.
189 """
191 ConfigClass = PhotonTransferCurveExtractConfig
192 _DefaultName = 'cpPtcExtract'
194 def runQuantum(self, butlerQC, inputRefs, outputRefs):
195 """Ensure that the input and output dimensions are passed along.
197 Parameters
198 ----------
199 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
200 Butler to operate on.
201 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
202 Input data refs to load.
203 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
204 Output data refs to persist.
205 """
206 inputs = butlerQC.get(inputRefs)
207 # Ids of input list of exposure references
208 # (deferLoad=True in the input connections)
210 inputs['inputDims'] = [expRef.datasetRef.dataId['exposure'] for expRef in inputRefs.inputExp]
212 # Dictionary, keyed by expTime, with tuples containing flat
213 # exposures and their IDs.
214 if self.config.matchByExposureId:
215 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims'])
216 else:
217 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims'])
219 outputs = self.run(**inputs)
220 butlerQC.put(outputs, outputRefs)
222 def run(self, inputExp, inputDims):
223 """Measure covariances from difference of flat pairs
225 Parameters
226 ----------
227 inputExp : `dict` [`float`, `list`
228 [`~lsst.pipe.base.connections.DeferredDatasetRef`]]
229 Dictionary that groups references to flat-field exposures that
230 have the same exposure time (seconds), or that groups them
231 sequentially by their exposure id.
233 inputDims : `list`
234 List of exposure IDs.
236 Returns
237 -------
238 results : `lsst.pipe.base.Struct`
239 The results struct containing:
241 ``outputCovariances``
242 A list containing the per-pair PTC measurements (`list`
243 [`lsst.ip.isr.PhotonTransferCurveDataset`])
244 """
245 # inputExp.values() returns a view, which we turn into a list. We then
246 # access the first exposure-ID tuple to get the detector.
247 # The first "get()" retrieves the exposure from the exposure reference.
248 detector = list(inputExp.values())[0][0][0].get(component='detector')
249 detNum = detector.getId()
250 amps = detector.getAmplifiers()
251 ampNames = [amp.getName() for amp in amps]
253 # Each amp may have a different min and max ADU signal
254 # specified in the config.
255 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
256 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
257 for ampName in ampNames:
258 if 'ALL_AMPS' in self.config.maxMeanSignal:
259 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
260 elif ampName in self.config.maxMeanSignal:
261 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
263 if 'ALL_AMPS' in self.config.minMeanSignal:
264 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
265 elif ampName in self.config.minMeanSignal:
266 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
267 # These are the column names for `tupleRows` below.
268 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
269 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
270 # Create a dummy ptcDataset
271 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
272 self.config.maximumRangeCovariancesAstier)
273 # Initialize amps of `dummyPtcDatset`.
274 for ampName in ampNames:
275 dummyPtcDataset.setAmpValues(ampName)
276 # Output list with PTC datasets.
277 partialPtcDatasetList = []
278 # The number of output references needs to match that of input
279 # references: initialize outputlist with dummy PTC datasets.
280 for i in range(len(inputDims)):
281 partialPtcDatasetList.append(dummyPtcDataset)
283 if self.config.numEdgeSuspect > 0:
284 isrTask = IsrTask()
285 self.log.info("Masking %d pixels from the edges of all exposures as SUSPECT.",
286 self.config.numEdgeSuspect)
288 for expTime in inputExp:
289 exposures = inputExp[expTime]
290 if len(exposures) == 1:
291 self.log.warning("Only one exposure found at expTime %f. Dropping exposure %d.",
292 expTime, exposures[0][1])
293 continue
294 else:
295 # Only use the first two exposures at expTime. Each
296 # elements is a tuple (exposure, expId)
297 expRef1, expId1 = exposures[0]
298 expRef2, expId2 = exposures[1]
299 # use get() to obtain `lsst.afw.image.Exposure`
300 exp1, exp2 = expRef1.get(), expRef2.get()
302 if len(exposures) > 2:
303 self.log.warning("Already found 2 exposures at expTime %f. Ignoring exposures: %s",
304 expTime, ", ".join(str(i[1]) for i in exposures[2:]))
305 # Mask pixels at the edge of the detector or of each amp
306 if self.config.numEdgeSuspect > 0:
307 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
308 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
309 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
310 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
312 nAmpsNan = 0
313 partialPtcDataset = PhotonTransferCurveDataset(ampNames, 'PARTIAL',
314 self.config.maximumRangeCovariancesAstier)
315 for ampNumber, amp in enumerate(detector):
316 ampName = amp.getName()
317 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for
318 # (i,j) in {maxLag, maxLag}^2]
319 doRealSpace = self.config.covAstierRealSpace
320 if self.config.detectorMeasurementRegion == 'AMP':
321 region = amp.getBBox()
322 elif self.config.detectorMeasurementRegion == 'FULL':
323 region = None
324 # `measureMeanVarCov` is the function that measures
325 # the variance and covariances from the difference
326 # image of two flats at the same exposure time. The
327 # variable `covAstier` is of the form: [(i, j, var
328 # (cov[0,0]), cov, npix) for (i,j) in {maxLag,
329 # maxLag}^2]
330 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region,
331 covAstierRealSpace=doRealSpace)
332 # Correction factor for sigma clipping. Function
333 # returns 1/sqrt(varFactor), so it needs to be
334 # squared. varDiff is calculated via
335 # afwMath.VARIANCECLIP.
336 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2
337 varDiff *= varFactor
339 expIdMask = True
340 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
341 msg = ("NaN mean or var, or None cov in amp %s in exposure pair %d, %d of detector %d.",
342 ampName, expId1, expId2, detNum)
343 self.log.warning(msg)
344 nAmpsNan += 1
345 expIdMask = False
346 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
347 self.config.maximumRangeCovariancesAstier), np.nan)
348 covSqrtWeights = np.full_like(covArray, np.nan)
350 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
351 expIdMask = False
353 if covAstier is not None:
354 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
355 ampName) for covRow in covAstier]
356 tempStructArray = np.array(tupleRows, dtype=tags)
357 covArray, vcov, _ = makeCovArray(tempStructArray,
358 self.config.maximumRangeCovariancesAstier)
359 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
361 # Correct covArray for sigma clipping:
362 # 1) Apply varFactor twice for the whole covariance matrix
363 covArray *= varFactor**2
364 # 2) But, only once for the variance element of the
365 # matrix, covArray[0,0]
366 covArray[0, 0] /= varFactor
368 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff],
369 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)],
370 expIdMask=[expIdMask], covArray=covArray,
371 covSqrtWeights=covSqrtWeights)
372 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
373 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple
374 # with a single-element array, so [0][0]
375 # is necessary to extract the required index.
376 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
377 partialPtcDatasetList[datasetIndex] = partialPtcDataset
379 if nAmpsNan == len(ampNames):
380 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
381 self.log.warning(msg)
382 return pipeBase.Struct(
383 outputCovariances=partialPtcDatasetList,
384 )
386 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
387 """Calculate the mean of each of two exposures and the variance
388 and covariance of their difference. The variance is calculated
389 via afwMath, and the covariance via the methods in Astier+19
390 (appendix A). In theory, var = covariance[0,0]. This should
391 be validated, and in the future, we may decide to just keep
392 one (covariance).
394 Parameters
395 ----------
396 exposure1 : `lsst.afw.image.exposure.ExposureF`
397 First exposure of flat field pair.
398 exposure2 : `lsst.afw.image.exposure.ExposureF`
399 Second exposure of flat field pair.
400 region : `lsst.geom.Box2I`, optional
401 Region of each exposure where to perform the calculations
402 (e.g, an amplifier).
403 covAstierRealSpace : `bool`, optional
404 Should the covariannces in Astier+19 be calculated in real
405 space or via FFT? See Appendix A of Astier+19.
407 Returns
408 -------
409 mu : `float` or `NaN`
410 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means
411 of the regions in both exposures. If either mu1 or m2 are
412 NaN's, the returned value is NaN.
413 varDiff : `float` or `NaN`
414 Half of the clipped variance of the difference of the
415 regions inthe two input exposures. If either mu1 or m2 are
416 NaN's, the returned value is NaN.
417 covDiffAstier : `list` or `NaN`
418 List with tuples of the form (dx, dy, var, cov, npix), where:
419 dx : `int`
420 Lag in x
421 dy : `int`
422 Lag in y
423 var : `float`
424 Variance at (dx, dy).
425 cov : `float`
426 Covariance at (dx, dy).
427 nPix : `int`
428 Number of pixel pairs used to evaluate var and cov.
430 If either mu1 or m2 are NaN's, the returned value is NaN.
431 """
432 if region is not None:
433 im1Area = exposure1.maskedImage[region]
434 im2Area = exposure2.maskedImage[region]
435 else:
436 im1Area = exposure1.maskedImage
437 im2Area = exposure2.maskedImage
439 if self.config.binSize > 1:
440 im1Area = afwMath.binImage(im1Area, self.config.binSize)
441 im2Area = afwMath.binImage(im2Area, self.config.binSize)
443 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
444 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
445 self.config.nIterSigmaClipPtc,
446 im1MaskVal)
447 im1StatsCtrl.setNanSafe(True)
448 im1StatsCtrl.setAndMask(im1MaskVal)
450 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
451 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
452 self.config.nIterSigmaClipPtc,
453 im2MaskVal)
454 im2StatsCtrl.setNanSafe(True)
455 im2StatsCtrl.setAndMask(im2MaskVal)
457 # Clipped mean of images; then average of mean.
458 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
459 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
460 if np.isnan(mu1) or np.isnan(mu2):
461 self.log.warning("Mean of amp in image 1 or 2 is NaN: %f, %f.", mu1, mu2)
462 return np.nan, np.nan, None
463 mu = 0.5*(mu1 + mu2)
465 # Take difference of pairs
466 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
467 temp = im2Area.clone()
468 temp *= mu1
469 diffIm = im1Area.clone()
470 diffIm *= mu2
471 diffIm -= temp
472 diffIm /= mu
474 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
475 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
476 self.config.nIterSigmaClipPtc,
477 diffImMaskVal)
478 diffImStatsCtrl.setNanSafe(True)
479 diffImStatsCtrl.setAndMask(diffImMaskVal)
481 # Variance calculation via afwMath
482 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
484 # Covariances calculations
485 # Get the pixels that were not clipped
486 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()
487 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, diffImStatsCtrl).getValue()
488 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
489 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
491 # Get the pixels in the mask planes of the difference image
492 # that were ignored by the clipping algorithm
493 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
494 # Combine the two sets of pixels ('1': use; '0': don't use)
495 # into a final weight matrix to be used in the covariance
496 # calculations below.
497 w = unmasked*wDiff
499 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance:
500 self.log.warning("Number of good points for covariance calculation (%s) is less "
501 "(than threshold %s)", np.sum(w), self.config.minNumberGoodPixelsForCovariance)
502 return np.nan, np.nan, None
504 maxRangeCov = self.config.maximumRangeCovariancesAstier
505 if covAstierRealSpace:
506 # Calculate covariances in real space.
507 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov)
508 else:
509 # Calculate covariances via FFT (default).
510 shapeDiff = np.array(diffIm.image.array.shape)
511 # Calculate the sizes of FFT dimensions.
512 s = shapeDiff + maxRangeCov
513 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
514 fftSize = np.array(2**(tempSize+1)).astype(int)
515 fftShape = (fftSize[0], fftSize[1])
517 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
518 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
520 # Compare Cov[0,0] and afwMath.VARIANCECLIP covDiffAstier[0]
521 # is the Cov[0,0] element, [3] is the variance, and there's a
522 # factor of 0.5 difference with afwMath.VARIANCECLIP.
523 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
524 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
525 if fractionalDiff >= thresholdPercentage:
526 self.log.warning("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
527 "is more than %f%%: %f", thresholdPercentage, fractionalDiff)
529 return mu, varDiff, covDiffAstier