Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.afw.math as afwMath
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27from lsst.cp.pipe.utils import arrangeFlatsByExpTime, arrangeFlatsByExpId
29import lsst.pipe.base.connectionTypes as cT
31from .astierCovPtcUtils import (CovFastFourierTransform, computeCovDirect)
32from .astierCovPtcFit import makeCovArray
34from lsst.ip.isr import PhotonTransferCurveDataset
37__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
40class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
41 dimensions=("instrument", "detector")):
43 inputExp = cT.Input(
44 name="ptcInputExposurePairs",
45 doc="Input post-ISR processed exposure pairs (flats) to"
46 "measure covariances from.",
47 storageClass="Exposure",
48 dimensions=("instrument", "exposure", "detector"),
49 multiple=True,
50 deferLoad=False,
51 )
53 outputCovariances = cT.Output(
54 name="ptcCovariances",
55 doc="Extracted flat (co)variances.",
56 storageClass="PhotonTransferCurveDataset",
57 dimensions=("instrument", "exposure", "detector"),
58 multiple=True,
59 )
62class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
63 pipelineConnections=PhotonTransferCurveExtractConnections):
64 """Configuration for the measurement of covariances from flats.
65 """
66 matchByExposureId = pexConfig.Field(
67 dtype=bool,
68 doc="Should exposures by matched by ID rather than exposure time?",
69 default=False,
70 )
71 maximumRangeCovariancesAstier = pexConfig.Field(
72 dtype=int,
73 doc="Maximum range of covariances as in Astier+19",
74 default=8,
75 )
76 covAstierRealSpace = pexConfig.Field(
77 dtype=bool,
78 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).",
79 default=False,
80 )
81 binSize = pexConfig.Field(
82 dtype=int,
83 doc="Bin the image by this factor in both dimensions.",
84 default=1,
85 )
86 minMeanSignal = pexConfig.DictField(
87 keytype=str,
88 itemtype=float,
89 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp."
90 " The same cut is applied to all amps if this dictionary is of the form"
91 " {'ALL_AMPS': value}",
92 default={'ALL_AMPS': 0.0},
93 )
94 maxMeanSignal = pexConfig.DictField(
95 keytype=str,
96 itemtype=float,
97 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
98 " The same cut is applied to all amps if this dictionary is of the form"
99 " {'ALL_AMPS': value}",
100 default={'ALL_AMPS': 1e6},
101 )
102 maskNameList = pexConfig.ListField(
103 dtype=str,
104 doc="Mask list to exclude from statistics calculations.",
105 default=['SUSPECT', 'BAD', 'NO_DATA'],
106 )
107 nSigmaClipPtc = pexConfig.Field(
108 dtype=float,
109 doc="Sigma cut for afwMath.StatisticsControl()",
110 default=5.5,
111 )
112 nIterSigmaClipPtc = pexConfig.Field(
113 dtype=int,
114 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
115 default=1,
116 )
117 minNumberGoodPixelsForCovariance = pexConfig.Field(
118 dtype=int,
119 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
120 " direclty).",
121 default=10000,
122 )
123 detectorMeasurementRegion = pexConfig.ChoiceField(
124 dtype=str,
125 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
126 default='AMP',
127 allowed={
128 "AMP": "Amplifier of the detector.",
129 "FULL": "Full image."
130 }
131 )
134class PhotonTransferCurveExtractTask(pipeBase.PipelineTask,
135 pipeBase.CmdLineTask):
136 """Task to measure covariances from flat fields.
137 This task receives as input a list of flat-field images
138 (flats), and sorts these flats in pairs taken at the
139 same time (if there's a different number of flats,
140 those flats are discarded). The mean, variance, and
141 covariances are measured from the difference of the flat
142 pairs at a given time. The variance is calculated
143 via afwMath, and the covariance via the methods in Astier+19
144 (appendix A). In theory, var = covariance[0,0]. This should
145 be validated, and in the future, we may decide to just keep
146 one (covariance).
148 The measured covariances at a particular time (along with
149 other quantities such as the mean) are stored in a PTC dataset
150 object (`PhotonTransferCurveDataset`), which gets partially
151 filled. The number of partially-filled PTC dataset objects
152 will be less than the number of input exposures, but gen3
153 requires/assumes that the number of input dimensions matches
154 bijectively the number of output dimensions. Therefore, a
155 number of "dummy" PTC dataset are inserted in the output list
156 that has the partially-filled PTC datasets with the covariances.
157 This output list will be used as input of
158 `PhotonTransferCurveSolveTask`, which will assemble the multiple
159 `PhotonTransferCurveDataset`s into a single one in order to fit
160 the measured covariances as a function of flux to a particular
161 model.
163 Astier+19: "The Shape of the Photon Transfer Curve of CCD
164 sensors", arXiv:1905.08677.
165 """
166 ConfigClass = PhotonTransferCurveExtractConfig
167 _DefaultName = 'cpPtcExtract'
169 def runQuantum(self, butlerQC, inputRefs, outputRefs):
170 """Ensure that the input and output dimensions are passed along.
172 Parameters
173 ----------
174 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
175 Butler to operate on.
176 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
177 Input data refs to load.
178 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
179 Output data refs to persist.
180 """
181 inputs = butlerQC.get(inputRefs)
182 # Dictionary, keyed by expTime, with flat exposures
183 if self.config.matchByExposureId:
184 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'])
185 else:
186 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'])
187 # Ids of input list of exposures
188 inputs['inputDims'] = [expId.dataId['exposure'] for expId in inputRefs.inputExp]
189 outputs = self.run(**inputs)
190 butlerQC.put(outputs, outputRefs)
192 def run(self, inputExp, inputDims):
193 """Measure covariances from difference of flat pairs
195 Parameters
196 ----------
197 inputExp : `dict` [`float`,
198 (`~lsst.afw.image.exposure.exposure.ExposureF`,
199 `~lsst.afw.image.exposure.exposure.ExposureF`, ...,
200 `~lsst.afw.image.exposure.exposure.ExposureF`)]
201 Dictionary that groups flat-field exposures that have the same
202 exposure time (seconds).
204 inputDims : `list`
205 List of exposure IDs.
206 """
207 # inputExp.values() returns a view, which we turn into a list. We then
208 # access the first exposure to get teh detector.
209 detector = list(inputExp.values())[0][0].getDetector()
210 detNum = detector.getId()
211 amps = detector.getAmplifiers()
212 ampNames = [amp.getName() for amp in amps]
214 # Each amp may have a different min and max ADU signal specified in the config.
215 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
216 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
217 for ampName in ampNames:
218 if 'ALL_AMPS' in self.config.maxMeanSignal:
219 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
220 elif ampName in self.config.maxMeanSignal:
221 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
223 if 'ALL_AMPS' in self.config.minMeanSignal:
224 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
225 elif ampName in self.config.minMeanSignal:
226 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
227 # These are the column names for `tupleRows` below.
228 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
229 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
230 # Create a dummy ptcDataset
231 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
232 self.config.maximumRangeCovariancesAstier)
233 # Initialize amps of `dummyPtcDatset`.
234 for ampName in ampNames:
235 dummyPtcDataset.setAmpValues(ampName)
236 # Output list with PTC datasets.
237 partialPtcDatasetList = []
238 # The number of output references needs to match that of input references:
239 # initialize outputlist with dummy PTC datasets.
240 for i in range(len(inputDims)):
241 partialPtcDatasetList.append(dummyPtcDataset)
243 for expTime in inputExp:
244 exposures = inputExp[expTime]
245 if len(exposures) == 1:
246 self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure "
247 f"{exposures[0].getInfo().getVisitInfo().getExposureId()}.")
248 continue
249 else:
250 # Only use the first two exposures at expTime
251 exp1, exp2 = exposures[0], exposures[1]
252 if len(exposures) > 2:
253 self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
254 "Ignoring exposures: "
255 f"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}")
256 expId1 = exp1.getInfo().getVisitInfo().getExposureId()
257 expId2 = exp2.getInfo().getVisitInfo().getExposureId()
258 nAmpsNan = 0
259 partialPtcDataset = PhotonTransferCurveDataset(ampNames, '',
260 self.config.maximumRangeCovariancesAstier)
261 for ampNumber, amp in enumerate(detector):
262 ampName = amp.getName()
263 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for (i,j) in {maxLag, maxLag}^2]
264 doRealSpace = self.config.covAstierRealSpace
265 if self.config.detectorMeasurementRegion == 'AMP':
266 region = amp.getBBox()
267 elif self.config.detectorMeasurementRegion == 'FULL':
268 region = None
269 # `measureMeanVarCov` is the function that measures the variance and covariances from
270 # the difference image of two flats at the same exposure time.
271 # The variable `covAstier` is of the form: [(i, j, var (cov[0,0]), cov, npix) for (i,j)
272 # in {maxLag, maxLag}^2]
273 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region,
274 covAstierRealSpace=doRealSpace)
275 expIdMask = True
276 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
277 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
278 f" {expId2} of detector {detNum}.")
279 self.log.warn(msg)
280 nAmpsNan += 1
281 expIdMask = False
282 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
283 self.config.maximumRangeCovariancesAstier), np.nan)
284 covSqrtWeights = np.full_like(covArray, np.nan)
286 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
287 expIdMask = False
289 if covAstier is not None:
290 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
291 ampName) for covRow in covAstier]
292 tempStructArray = np.array(tupleRows, dtype=tags)
293 covArray, vcov, _ = makeCovArray(tempStructArray,
294 self.config.maximumRangeCovariancesAstier)
295 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
297 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff],
298 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)],
299 expIdMask=[expIdMask], covArray=covArray,
300 covSqrtWeights=covSqrtWeights)
301 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
302 # expId1 and expId2, as returned by getInfo().getVisitInfo().getExposureId(),
303 # and the exposure IDs stured in inoutDims,
304 # may have the zero-padded detector number appended at
305 # the end (in gen3). A temporary fix is to consider expId//1000 and/or
306 # inputDims//1000.
307 # Below, np.where(expId1 == np.array(inputDims)) (and the other analogous
308 # comparisons) returns a tuple with a single-element array, so [0][0]
309 # is necessary to extract the required index.
310 try:
311 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
312 except IndexError:
313 try:
314 datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0]
315 except IndexError:
316 datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0]
317 partialPtcDatasetList[datasetIndex] = partialPtcDataset
318 if nAmpsNan == len(ampNames):
319 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
320 self.log.warn(msg)
321 return pipeBase.Struct(
322 outputCovariances=partialPtcDatasetList,
323 )
325 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
326 """Calculate the mean of each of two exposures and the variance
327 and covariance of their difference. The variance is calculated
328 via afwMath, and the covariance via the methods in Astier+19
329 (appendix A). In theory, var = covariance[0,0]. This should
330 be validated, and in the future, we may decide to just keep
331 one (covariance).
333 Parameters
334 ----------
335 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
336 First exposure of flat field pair.
337 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
338 Second exposure of flat field pair.
339 region : `lsst.geom.Box2I`, optional
340 Region of each exposure where to perform the calculations (e.g, an amplifier).
341 covAstierRealSpace : `bool`, optional
342 Should the covariannces in Astier+19 be calculated in real space or via FFT?
343 See Appendix A of Astier+19.
345 Returns
346 -------
347 mu : `float` or `NaN`
348 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
349 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
350 varDiff : `float` or `NaN`
351 Half of the clipped variance of the difference of the regions inthe two input
352 exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
353 covDiffAstier : `list` or `NaN`
354 List with tuples of the form (dx, dy, var, cov, npix), where:
355 dx : `int`
356 Lag in x
357 dy : `int`
358 Lag in y
359 var : `float`
360 Variance at (dx, dy).
361 cov : `float`
362 Covariance at (dx, dy).
363 nPix : `int`
364 Number of pixel pairs used to evaluate var and cov.
365 If either mu1 or m2 are NaN's, the returned value is NaN.
366 """
368 if region is not None:
369 im1Area = exposure1.maskedImage[region]
370 im2Area = exposure2.maskedImage[region]
371 else:
372 im1Area = exposure1.maskedImage
373 im2Area = exposure2.maskedImage
375 if self.config.binSize > 1:
376 im1Area = afwMath.binImage(im1Area, self.config.binSize)
377 im2Area = afwMath.binImage(im2Area, self.config.binSize)
379 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
380 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
381 self.config.nIterSigmaClipPtc,
382 im1MaskVal)
383 im1StatsCtrl.setNanSafe(True)
384 im1StatsCtrl.setAndMask(im1MaskVal)
386 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
387 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
388 self.config.nIterSigmaClipPtc,
389 im2MaskVal)
390 im2StatsCtrl.setNanSafe(True)
391 im2StatsCtrl.setAndMask(im2MaskVal)
393 # Clipped mean of images; then average of mean.
394 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
395 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
396 if np.isnan(mu1) or np.isnan(mu2):
397 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
398 return np.nan, np.nan, None
399 mu = 0.5*(mu1 + mu2)
401 # Take difference of pairs
402 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
403 temp = im2Area.clone()
404 temp *= mu1
405 diffIm = im1Area.clone()
406 diffIm *= mu2
407 diffIm -= temp
408 diffIm /= mu
410 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
411 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
412 self.config.nIterSigmaClipPtc,
413 diffImMaskVal)
414 diffImStatsCtrl.setNanSafe(True)
415 diffImStatsCtrl.setAndMask(diffImMaskVal)
417 # Variance calculation via afwMath
418 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
420 # Covariances calculations
421 # Get the mask and identify good pixels as '1', and the rest as '0'.
422 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
423 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
425 w12 = w1*w2
426 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
427 w = w12*wDiff
429 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance:
430 self.log.warn(f"Number of good points for covariance calculation ({np.sum(w)}) is less "
431 f"(than threshold {self.config.minNumberGoodPixelsForCovariance})")
432 return np.nan, np.nan, None
434 maxRangeCov = self.config.maximumRangeCovariancesAstier
435 if covAstierRealSpace:
436 # Calculate covariances in real space.
437 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov)
438 else:
439 # Calculate covariances via FFT (default).
440 shapeDiff = np.array(diffIm.image.array.shape)
441 # Calculate the sizes of FFT dimensions.
442 s = shapeDiff + maxRangeCov
443 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
444 fftSize = np.array(2**(tempSize+1)).astype(int)
445 fftShape = (fftSize[0], fftSize[1])
447 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
448 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
450 return mu, varDiff, covDiffAstier