Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.afw.math as afwMath
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
28 sigmaClipCorrection)
30import lsst.pipe.base.connectionTypes as cT
32from .astierCovPtcUtils import (CovFastFourierTransform, computeCovDirect)
33from .astierCovPtcFit import makeCovArray
35from lsst.ip.isr import PhotonTransferCurveDataset
36from lsst.ip.isr import IsrTask
38__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
41class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "detector")):
44 inputExp = cT.Input(
45 name="ptcInputExposurePairs",
46 doc="Input post-ISR processed exposure pairs (flats) to"
47 "measure covariances from.",
48 storageClass="Exposure",
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 deferLoad=False,
52 )
54 outputCovariances = cT.Output(
55 name="ptcCovariances",
56 doc="Extracted flat (co)variances.",
57 storageClass="PhotonTransferCurveDataset",
58 dimensions=("instrument", "exposure", "detector"),
59 multiple=True,
60 )
63class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
64 pipelineConnections=PhotonTransferCurveExtractConnections):
65 """Configuration for the measurement of covariances from flats.
66 """
67 matchByExposureId = pexConfig.Field(
68 dtype=bool,
69 doc="Should exposures by matched by ID rather than exposure time?",
70 default=False,
71 )
72 maximumRangeCovariancesAstier = pexConfig.Field(
73 dtype=int,
74 doc="Maximum range of covariances as in Astier+19",
75 default=8,
76 )
77 covAstierRealSpace = pexConfig.Field(
78 dtype=bool,
79 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
80 default=False,
81 )
82 binSize = pexConfig.Field(
83 dtype=int,
84 doc="Bin the image by this factor in both dimensions.",
85 default=1,
86 )
87 minMeanSignal = pexConfig.DictField(
88 keytype=str,
89 itemtype=float,
90 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
91 " The same cut is applied to all amps if this dictionary is of the form"
92 " {'ALL_AMPS': value}",
93 default={'ALL_AMPS': 0.0},
94 )
95 maxMeanSignal = pexConfig.DictField(
96 keytype=str,
97 itemtype=float,
98 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
99 " The same cut is applied to all amps if this dictionary is of the form"
100 " {'ALL_AMPS': value}",
101 default={'ALL_AMPS': 1e6},
102 )
103 maskNameList = pexConfig.ListField(
104 dtype=str,
105 doc="Mask list to exclude from statistics calculations.",
106 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'],
107 )
108 nSigmaClipPtc = pexConfig.Field(
109 dtype=float,
110 doc="Sigma cut for afwMath.StatisticsControl()",
111 default=5.5,
112 )
113 nIterSigmaClipPtc = pexConfig.Field(
114 dtype=int,
115 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
116 default=3,
117 )
118 minNumberGoodPixelsForCovariance = pexConfig.Field(
119 dtype=int,
120 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
121 " direclty).",
122 default=10000,
123 )
124 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
125 dtype=float,
126 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
127 "for a region of a difference image is greater than this threshold (percentage), "
128 "a warning will be issued.",
129 default=1.,
130 )
131 detectorMeasurementRegion = pexConfig.ChoiceField(
132 dtype=str,
133 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
134 default='AMP',
135 allowed={
136 "AMP": "Amplifier of the detector.",
137 "FULL": "Full image."
138 }
139 )
140 numEdgeSuspect = pexConfig.Field(
141 dtype=int,
142 doc="Number of edge pixels to be flagged as untrustworthy.",
143 default=0,
144 )
145 edgeMaskLevel = pexConfig.ChoiceField(
146 dtype=str,
147 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
148 default="DETECTOR",
149 allowed={
150 'DETECTOR': 'Mask only the edges of the full detector.',
151 'AMP': 'Mask edges of each amplifier.',
152 },
153 )
156class PhotonTransferCurveExtractTask(pipeBase.PipelineTask,
157 pipeBase.CmdLineTask):
158 """Task to measure covariances from flat fields.
159 This task receives as input a list of flat-field images
160 (flats), and sorts these flats in pairs taken at the
161 same time (if there's a different number of flats,
162 those flats are discarded). The mean, variance, and
163 covariances are measured from the difference of the flat
164 pairs at a given time. The variance is calculated
165 via afwMath, and the covariance via the methods in Astier+19
166 (appendix A). In theory, var = covariance[0,0]. This should
167 be validated, and in the future, we may decide to just keep
168 one (covariance).
170 The measured covariances at a particular time (along with
171 other quantities such as the mean) are stored in a PTC dataset
172 object (`PhotonTransferCurveDataset`), which gets partially
173 filled. The number of partially-filled PTC dataset objects
174 will be less than the number of input exposures, but gen3
175 requires/assumes that the number of input dimensions matches
176 bijectively the number of output dimensions. Therefore, a
177 number of "dummy" PTC dataset are inserted in the output list
178 that has the partially-filled PTC datasets with the covariances.
179 This output list will be used as input of
180 `PhotonTransferCurveSolveTask`, which will assemble the multiple
181 `PhotonTransferCurveDataset`s into a single one in order to fit
182 the measured covariances as a function of flux to a particular
183 model.
185 Astier+19: "The Shape of the Photon Transfer Curve of CCD
186 sensors", arXiv:1905.08677.
187 """
188 ConfigClass = PhotonTransferCurveExtractConfig
189 _DefaultName = 'cpPtcExtract'
191 def runQuantum(self, butlerQC, inputRefs, outputRefs):
192 """Ensure that the input and output dimensions are passed along.
194 Parameters
195 ----------
196 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
197 Butler to operate on.
198 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
199 Input data refs to load.
200 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
201 Output data refs to persist.
202 """
203 inputs = butlerQC.get(inputRefs)
204 # Dictionary, keyed by expTime, with flat exposures
205 if self.config.matchByExposureId:
206 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'])
207 else:
208 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'])
209 # Ids of input list of exposures
210 inputs['inputDims'] = [expId.dataId['exposure'] for expId in inputRefs.inputExp]
211 outputs = self.run(**inputs)
212 butlerQC.put(outputs, outputRefs)
214 def run(self, inputExp, inputDims):
215 """Measure covariances from difference of flat pairs
217 Parameters
218 ----------
219 inputExp : `dict` [`float`,
220 (`~lsst.afw.image.exposure.exposure.ExposureF`,
221 `~lsst.afw.image.exposure.exposure.ExposureF`, ...,
222 `~lsst.afw.image.exposure.exposure.ExposureF`)]
223 Dictionary that groups flat-field exposures that have the same
224 exposure time (seconds).
226 inputDims : `list`
227 List of exposure IDs.
228 """
229 # inputExp.values() returns a view, which we turn into a list. We then
230 # access the first exposure to get teh detector.
231 detector = list(inputExp.values())[0][0].getDetector()
232 detNum = detector.getId()
233 amps = detector.getAmplifiers()
234 ampNames = [amp.getName() for amp in amps]
236 # Each amp may have a different min and max ADU signal specified in the config.
237 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
238 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
239 for ampName in ampNames:
240 if 'ALL_AMPS' in self.config.maxMeanSignal:
241 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
242 elif ampName in self.config.maxMeanSignal:
243 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
245 if 'ALL_AMPS' in self.config.minMeanSignal:
246 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
247 elif ampName in self.config.minMeanSignal:
248 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
249 # These are the column names for `tupleRows` below.
250 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
251 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
252 # Create a dummy ptcDataset
253 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
254 self.config.maximumRangeCovariancesAstier)
255 # Initialize amps of `dummyPtcDatset`.
256 for ampName in ampNames:
257 dummyPtcDataset.setAmpValues(ampName)
258 # Output list with PTC datasets.
259 partialPtcDatasetList = []
260 # The number of output references needs to match that of input references:
261 # initialize outputlist with dummy PTC datasets.
262 for i in range(len(inputDims)):
263 partialPtcDatasetList.append(dummyPtcDataset)
265 if self.config.numEdgeSuspect > 0:
266 isrTask = IsrTask()
267 self.log.info(f"Masking {self.config.numEdgeSuspect} pixels from the edges "
268 "of all exposures as SUSPECT.")
270 for expTime in inputExp:
271 exposures = inputExp[expTime]
272 if len(exposures) == 1:
273 self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure "
274 f"{exposures[0].getInfo().getVisitInfo().getExposureId()}.")
275 continue
276 else:
277 # Only use the first two exposures at expTime
278 exp1, exp2 = exposures[0], exposures[1]
279 if len(exposures) > 2:
280 self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
281 "Ignoring exposures: "
282 f"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}")
283 # Mask pixels at the edge of the detector or of each amp
284 if self.config.numEdgeSuspect > 0:
285 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
286 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
287 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
288 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
289 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
290 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
291 nAmpsNan = 0
292 partialPtcDataset = PhotonTransferCurveDataset(ampNames, '',
293 self.config.maximumRangeCovariancesAstier)
294 for ampNumber, amp in enumerate(detector):
295 ampName = amp.getName()
296 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for (i,j) in {maxLag, maxLag}^2]
297 doRealSpace = self.config.covAstierRealSpace
298 if self.config.detectorMeasurementRegion == 'AMP':
299 region = amp.getBBox()
300 elif self.config.detectorMeasurementRegion == 'FULL':
301 region = None
302 # `measureMeanVarCov` is the function that measures the variance and covariances from
303 # the difference image of two flats at the same exposure time.
304 # The variable `covAstier` is of the form: [(i, j, var (cov[0,0]), cov, npix) for (i,j)
305 # in {maxLag, maxLag}^2]
306 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region,
307 covAstierRealSpace=doRealSpace)
308 # Correction factor for sigma clipping. Function returns 1/sqrt(varFactor),
309 # so it needs to be squared. varDiff is calculated via afwMath.VARIANCECLIP.
310 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2
311 varDiff *= varFactor
313 expIdMask = True
314 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
315 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
316 f" {expId2} of detector {detNum}.")
317 self.log.warn(msg)
318 nAmpsNan += 1
319 expIdMask = False
320 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
321 self.config.maximumRangeCovariancesAstier), np.nan)
322 covSqrtWeights = np.full_like(covArray, np.nan)
324 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
325 expIdMask = False
327 if covAstier is not None:
328 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
329 ampName) for covRow in covAstier]
330 tempStructArray = np.array(tupleRows, dtype=tags)
331 covArray, vcov, _ = makeCovArray(tempStructArray,
332 self.config.maximumRangeCovariancesAstier)
333 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
335 # Correct covArray for sigma clipping:
336 # 1) Apply varFactor twice for the whole covariance matrix
337 covArray *= varFactor**2
338 # 2) But, only once for the variance element of the matrix, covArray[0,0]
339 covArray[0, 0] /= varFactor
341 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff],
342 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)],
343 expIdMask=[expIdMask], covArray=covArray,
344 covSqrtWeights=covSqrtWeights)
345 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
346 # expId1 and expId2, as returned by getInfo().getVisitInfo().getExposureId(),
347 # and the exposure IDs stured in inoutDims,
348 # may have the zero-padded detector number appended at
349 # the end (in gen3). A temporary fix is to consider expId//1000 and/or
350 # inputDims//1000.
351 # Below, np.where(expId1 == np.array(inputDims)) (and the other analogous
352 # comparisons) returns a tuple with a single-element array, so [0][0]
353 # is necessary to extract the required index.
354 try:
355 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
356 except IndexError:
357 try:
358 datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0]
359 except IndexError:
360 datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0]
361 partialPtcDatasetList[datasetIndex] = partialPtcDataset
362 if nAmpsNan == len(ampNames):
363 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
364 self.log.warn(msg)
365 return pipeBase.Struct(
366 outputCovariances=partialPtcDatasetList,
367 )
369 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
370 """Calculate the mean of each of two exposures and the variance
371 and covariance of their difference. The variance is calculated
372 via afwMath, and the covariance via the methods in Astier+19
373 (appendix A). In theory, var = covariance[0,0]. This should
374 be validated, and in the future, we may decide to just keep
375 one (covariance).
377 Parameters
378 ----------
379 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
380 First exposure of flat field pair.
381 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
382 Second exposure of flat field pair.
383 region : `lsst.geom.Box2I`, optional
384 Region of each exposure where to perform the calculations (e.g, an amplifier).
385 covAstierRealSpace : `bool`, optional
386 Should the covariannces in Astier+19 be calculated in real space or via FFT?
387 See Appendix A of Astier+19.
389 Returns
390 -------
391 mu : `float` or `NaN`
392 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
393 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
394 varDiff : `float` or `NaN`
395 Half of the clipped variance of the difference of the regions inthe two input
396 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
397 covDiffAstier : `list` or `NaN`
398 List with tuples of the form (dx, dy, var, cov, npix), where:
399 dx : `int`
400 Lag in x
401 dy : `int`
402 Lag in y
403 var : `float`
404 Variance at (dx, dy).
405 cov : `float`
406 Covariance at (dx, dy).
407 nPix : `int`
408 Number of pixel pairs used to evaluate var and cov.
409 If either mu1 or m2 are NaN's, the returned value is NaN.
410 """
412 if region is not None:
413 im1Area = exposure1.maskedImage[region]
414 im2Area = exposure2.maskedImage[region]
415 else:
416 im1Area = exposure1.maskedImage
417 im2Area = exposure2.maskedImage
419 if self.config.binSize > 1:
420 im1Area = afwMath.binImage(im1Area, self.config.binSize)
421 im2Area = afwMath.binImage(im2Area, self.config.binSize)
423 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
424 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
425 self.config.nIterSigmaClipPtc,
426 im1MaskVal)
427 im1StatsCtrl.setNanSafe(True)
428 im1StatsCtrl.setAndMask(im1MaskVal)
430 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
431 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
432 self.config.nIterSigmaClipPtc,
433 im2MaskVal)
434 im2StatsCtrl.setNanSafe(True)
435 im2StatsCtrl.setAndMask(im2MaskVal)
437 # Clipped mean of images; then average of mean.
438 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
439 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
440 if np.isnan(mu1) or np.isnan(mu2):
441 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
442 return np.nan, np.nan, None
443 mu = 0.5*(mu1 + mu2)
445 # Take difference of pairs
446 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
447 temp = im2Area.clone()
448 temp *= mu1
449 diffIm = im1Area.clone()
450 diffIm *= mu2
451 diffIm -= temp
452 diffIm /= mu
454 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
455 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
456 self.config.nIterSigmaClipPtc,
457 diffImMaskVal)
458 diffImStatsCtrl.setNanSafe(True)
459 diffImStatsCtrl.setAndMask(diffImMaskVal)
461 # Variance calculation via afwMath
462 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
464 # Covariances calculations
465 # Get the pixels that were not clipped
466 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()
467 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, diffImStatsCtrl).getValue()
468 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
469 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
471 # Get the pixels in the mask planes of teh differenc eimage that were ignored
472 # by the clipping algorithm
473 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
474 # Combine the two sets of pixels ('1': use; '0': don't use) into a final weight matrix
475 # to be used in the covariance calculations below.
476 w = unmasked*wDiff
478 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance:
479 self.log.warn(f"Number of good points for covariance calculation ({np.sum(w)}) is less "
480 f"(than threshold {self.config.minNumberGoodPixelsForCovariance})")
481 return np.nan, np.nan, None
483 maxRangeCov = self.config.maximumRangeCovariancesAstier
484 if covAstierRealSpace:
485 # Calculate covariances in real space.
486 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov)
487 else:
488 # Calculate covariances via FFT (default).
489 shapeDiff = np.array(diffIm.image.array.shape)
490 # Calculate the sizes of FFT dimensions.
491 s = shapeDiff + maxRangeCov
492 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
493 fftSize = np.array(2**(tempSize+1)).astype(int)
494 fftShape = (fftSize[0], fftSize[1])
496 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
497 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
499 # Compare Cov[0,0] and afwMath.VARIANCECLIP
500 # covDiffAstier[0] is the Cov[0,0] element, [3] is the variance, and there's a factor of 0.5
501 # difference with afwMath.VARIANCECLIP.
502 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
503 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
504 if fractionalDiff >= thresholdPercentage:
505 self.log.warn("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
506 f"is more than {thresholdPercentage}%: {fractionalDiff}")
508 return mu, varDiff, covDiffAstier