Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py: 11%
305 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-13 03:01 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-13 03:01 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from lmfit.models import GaussianModel
24import scipy.stats
26import lsst.afw.math as afwMath
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
30 arrangeFlatsByExpFlux, sigmaClipCorrection,
31 CovFastFourierTransform)
33import lsst.pipe.base.connectionTypes as cT
35from lsst.ip.isr import PhotonTransferCurveDataset
36from lsst.ip.isr import IsrTask
38__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
41class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "detector")):
44 inputExp = cT.Input(
45 name="ptcInputExposurePairs",
46 doc="Input post-ISR processed exposure pairs (flats) to"
47 "measure covariances from.",
48 storageClass="Exposure",
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 deferLoad=True,
52 )
53 taskMetadata = cT.Input(
54 name="isr_metadata",
55 doc="Input task metadata to extract statistics from.",
56 storageClass="TaskMetadata",
57 dimensions=("instrument", "exposure", "detector"),
58 multiple=True,
59 )
60 outputCovariances = cT.Output(
61 name="ptcCovariances",
62 doc="Extracted flat (co)variances.",
63 storageClass="PhotonTransferCurveDataset",
64 dimensions=("instrument", "exposure", "detector"),
65 isCalibration=True,
66 multiple=True,
67 )
70class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
71 pipelineConnections=PhotonTransferCurveExtractConnections):
72 """Configuration for the measurement of covariances from flats.
73 """
74 matchExposuresType = pexConfig.ChoiceField(
75 dtype=str,
76 doc="Match input exposures by time, flux, or expId",
77 default='TIME',
78 allowed={
79 "TIME": "Match exposures by exposure time.",
80 "FLUX": "Match exposures by target flux. Use header keyword"
81 " in matchExposuresByFluxKeyword to find the flux.",
82 "EXPID": "Match exposures by exposure ID."
83 }
84 )
85 matchExposuresByFluxKeyword = pexConfig.Field(
86 dtype=str,
87 doc="Header keyword for flux if matchExposuresType is FLUX.",
88 default='CCOBFLUX',
89 )
90 maximumRangeCovariancesAstier = pexConfig.Field(
91 dtype=int,
92 doc="Maximum range of covariances as in Astier+19",
93 default=8,
94 )
95 binSize = pexConfig.Field(
96 dtype=int,
97 doc="Bin the image by this factor in both dimensions.",
98 default=1,
99 )
100 minMeanSignal = pexConfig.DictField(
101 keytype=str,
102 itemtype=float,
103 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use."
104 " The same cut is applied to all amps if this parameter [`dict`] is passed as "
105 " {'ALL_AMPS': value}",
106 default={'ALL_AMPS': 0.0},
107 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.",
108 )
109 maxMeanSignal = pexConfig.DictField(
110 keytype=str,
111 itemtype=float,
112 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
113 " The same cut is applied to all amps if this dictionary is of the form"
114 " {'ALL_AMPS': value}",
115 default={'ALL_AMPS': 1e6},
116 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.",
117 )
118 maskNameList = pexConfig.ListField(
119 dtype=str,
120 doc="Mask list to exclude from statistics calculations.",
121 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'],
122 )
123 nSigmaClipPtc = pexConfig.Field(
124 dtype=float,
125 doc="Sigma cut for afwMath.StatisticsControl()",
126 default=5.5,
127 )
128 nIterSigmaClipPtc = pexConfig.Field(
129 dtype=int,
130 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
131 default=3,
132 )
133 minNumberGoodPixelsForCovariance = pexConfig.Field(
134 dtype=int,
135 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
136 " direclty).",
137 default=10000,
138 )
139 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
140 dtype=float,
141 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
142 "for a region of a difference image is greater than this threshold (percentage), "
143 "a warning will be issued.",
144 default=1.,
145 )
146 detectorMeasurementRegion = pexConfig.ChoiceField(
147 dtype=str,
148 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
149 default='AMP',
150 allowed={
151 "AMP": "Amplifier of the detector.",
152 "FULL": "Full image."
153 }
154 )
155 numEdgeSuspect = pexConfig.Field(
156 dtype=int,
157 doc="Number of edge pixels to be flagged as untrustworthy.",
158 default=0,
159 )
160 edgeMaskLevel = pexConfig.ChoiceField(
161 dtype=str,
162 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
163 default="DETECTOR",
164 allowed={
165 'DETECTOR': 'Mask only the edges of the full detector.',
166 'AMP': 'Mask edges of each amplifier.',
167 },
168 )
169 doGain = pexConfig.Field(
170 dtype=bool,
171 doc="Calculate a gain per input flat pair.",
172 default=True,
173 )
174 gainCorrectionType = pexConfig.ChoiceField(
175 dtype=str,
176 doc="Correction type for the gain.",
177 default='FULL',
178 allowed={
179 'NONE': 'No correction.',
180 'SIMPLE': 'First order correction.',
181 'FULL': 'Second order correction.'
182 }
183 )
184 ksHistNBins = pexConfig.Field(
185 dtype=int,
186 doc="Number of bins for the KS test histogram.",
187 default=100,
188 )
189 ksHistLimitMultiplier = pexConfig.Field(
190 dtype=float,
191 doc="Number of sigma (as predicted from the mean value) to compute KS test histogram.",
192 default=8.0,
193 )
194 ksHistMinDataValues = pexConfig.Field(
195 dtype=int,
196 doc="Minimum number of good data values to compute KS test histogram.",
197 default=100,
198 )
201class PhotonTransferCurveExtractTask(pipeBase.PipelineTask):
202 """Task to measure covariances from flat fields.
204 This task receives as input a list of flat-field images
205 (flats), and sorts these flats in pairs taken at the
206 same time (the task will raise if there is one one flat
207 at a given exposure time, and it will discard extra flats if
208 there are more than two per exposure time). This task measures
209 the mean, variance, and covariances from a region (e.g.,
210 an amplifier) of the difference image of the two flats with
211 the same exposure time (alternatively, all input images could have
212 the same exposure time but their flux changed).
214 The variance is calculated via afwMath, and the covariance
215 via the methods in Astier+19 (appendix A). In theory,
216 var = covariance[0,0]. This should be validated, and in the
217 future, we may decide to just keep one (covariance).
218 At this moment, if the two values differ by more than the value
219 of `thresholdDiffAfwVarVsCov00` (default: 1%), a warning will
220 be issued.
222 The measured covariances at a given exposure time (along with
223 other quantities such as the mean) are stored in a PTC dataset
224 object (`~lsst.ip.isr.PhotonTransferCurveDataset`), which gets
225 partially filled at this stage (the remainder of the attributes
226 of the dataset will be filled after running the second task of
227 the PTC-measurement pipeline, `~PhotonTransferCurveSolveTask`).
229 The number of partially-filled
230 `~lsst.ip.isr.PhotonTransferCurveDataset` objects will be less
231 than the number of input exposures because the task combines
232 input flats in pairs. However, it is required at this moment
233 that the number of input dimensions matches
234 bijectively the number of output dimensions. Therefore, a number
235 of "dummy" PTC datasets are inserted in the output list. This
236 output list will then be used as input of the next task in the
237 PTC-measurement pipeline, `PhotonTransferCurveSolveTask`,
238 which will assemble the multiple `PhotonTransferCurveDataset`
239 objects into a single one in order to fit the measured covariances
240 as a function of flux to one of three models
241 (see `PhotonTransferCurveSolveTask` for details).
243 Reference: Astier+19: "The Shape of the Photon Transfer Curve of CCD
244 sensors", arXiv:1905.08677.
245 """
247 ConfigClass = PhotonTransferCurveExtractConfig
248 _DefaultName = 'cpPtcExtract'
250 def runQuantum(self, butlerQC, inputRefs, outputRefs):
251 """Ensure that the input and output dimensions are passed along.
253 Parameters
254 ----------
255 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
256 Butler to operate on.
257 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection`
258 Input data refs to load.
259 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
260 Output data refs to persist.
261 """
262 inputs = butlerQC.get(inputRefs)
263 # Ids of input list of exposure references
264 # (deferLoad=True in the input connections)
265 inputs['inputDims'] = [expRef.datasetRef.dataId['exposure'] for expRef in inputRefs.inputExp]
267 # Dictionary, keyed by expTime (or expFlux or expId), with tuples
268 # containing flat exposures and their IDs.
269 matchType = self.config.matchExposuresType
270 if matchType == 'TIME':
271 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims'])
272 elif matchType == 'FLUX':
273 inputs['inputExp'] = arrangeFlatsByExpFlux(inputs['inputExp'], inputs['inputDims'],
274 self.config.matchExposuresByFluxKeyword)
275 else:
276 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims'])
278 outputs = self.run(**inputs)
279 outputs = self._guaranteeOutputs(inputs['inputDims'], outputs, outputRefs)
280 butlerQC.put(outputs, outputRefs)
282 def _guaranteeOutputs(self, inputDims, outputs, outputRefs):
283 """Ensure that all outputRefs have a matching output, and if they do
284 not, fill the output with dummy PTC datasets.
286 Parameters
287 ----------
288 inputDims : `dict` [`str`, `int`]
289 Input exposure dimensions.
290 outputs : `lsst.pipe.base.Struct`
291 Outputs from the ``run`` method. Contains the entry:
293 ``outputCovariances``
294 Output PTC datasets (`list` [`lsst.ip.isr.IsrCalib`])
295 outputRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection`
296 Container with all of the outputs expected to be generated.
298 Returns
299 -------
300 outputs : `lsst.pipe.base.Struct`
301 Dummy dataset padded version of the input ``outputs`` with
302 the same entries.
303 """
304 newCovariances = []
305 for ref in outputRefs.outputCovariances:
306 outputExpId = ref.dataId['exposure']
307 if outputExpId in inputDims:
308 entry = inputDims.index(outputExpId)
309 newCovariances.append(outputs.outputCovariances[entry])
310 else:
311 newPtc = PhotonTransferCurveDataset(['no amp'], 'DUMMY', 1)
312 newPtc.setAmpValuesPartialDataset('no amp')
313 newCovariances.append(newPtc)
314 return pipeBase.Struct(outputCovariances=newCovariances)
316 def run(self, inputExp, inputDims, taskMetadata):
318 """Measure covariances from difference of flat pairs
320 Parameters
321 ----------
322 inputExp : `dict` [`float`, `list`
323 [`~lsst.pipe.base.connections.DeferredDatasetRef`]]
324 Dictionary that groups references to flat-field exposures that
325 have the same exposure time (seconds), or that groups them
326 sequentially by their exposure id.
327 inputDims : `list`
328 List of exposure IDs.
329 taskMetadata : `list` [`lsst.pipe.base.TaskMetadata`]
330 List of exposures metadata from ISR.
332 Returns
333 -------
334 results : `lsst.pipe.base.Struct`
335 The resulting Struct contains:
337 ``outputCovariances``
338 A list containing the per-pair PTC measurements (`list`
339 [`lsst.ip.isr.PhotonTransferCurveDataset`])
340 """
341 # inputExp.values() returns a view, which we turn into a list. We then
342 # access the first exposure-ID tuple to get the detector.
343 # The first "get()" retrieves the exposure from the exposure reference.
344 detector = list(inputExp.values())[0][0][0].get(component='detector')
345 detNum = detector.getId()
346 amps = detector.getAmplifiers()
347 ampNames = [amp.getName() for amp in amps]
349 # Each amp may have a different min and max ADU signal
350 # specified in the config.
351 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
352 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
353 for ampName in ampNames:
354 if 'ALL_AMPS' in self.config.maxMeanSignal:
355 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
356 elif ampName in self.config.maxMeanSignal:
357 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
359 if 'ALL_AMPS' in self.config.minMeanSignal:
360 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
361 elif ampName in self.config.minMeanSignal:
362 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
363 # These are the column names for `tupleRows` below.
364 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
365 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
366 # Create a dummy ptcDataset. Dummy datasets will be
367 # used to ensure that the number of output and input
368 # dimensions match.
369 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
370 self.config.maximumRangeCovariancesAstier)
371 for ampName in ampNames:
372 dummyPtcDataset.setAmpValuesPartialDataset(ampName)
373 # Get read noise. Try from the exposure, then try
374 # taskMetadata. This adds a get() for the exposures.
375 readNoiseLists = {}
376 for pairIndex, expRefs in inputExp.items():
377 # This yields an index (exposure_time, seq_num, or flux)
378 # and a pair of references at that index.
379 for expRef, expId in expRefs:
380 # This yields an exposure ref and an exposureId.
381 exposureMetadata = expRef.get(component="metadata")
382 metadataIndex = inputDims.index(expId)
383 thisTaskMetadata = taskMetadata[metadataIndex]
385 for ampName in ampNames:
386 if ampName not in readNoiseLists:
387 readNoiseLists[ampName] = [self.getReadNoise(exposureMetadata,
388 thisTaskMetadata, ampName)]
389 else:
390 readNoiseLists[ampName].append(self.getReadNoise(exposureMetadata,
391 thisTaskMetadata, ampName))
393 readNoiseDict = {ampName: 0.0 for ampName in ampNames}
394 for ampName in ampNames:
395 # Take median read noise value
396 readNoiseDict[ampName] = np.nanmedian(readNoiseLists[ampName])
398 # Output list with PTC datasets.
399 partialPtcDatasetList = []
400 # The number of output references needs to match that of input
401 # references: initialize outputlist with dummy PTC datasets.
402 for i in range(len(inputDims)):
403 partialPtcDatasetList.append(dummyPtcDataset)
405 if self.config.numEdgeSuspect > 0:
406 isrTask = IsrTask()
407 self.log.info("Masking %d pixels from the edges of all %ss as SUSPECT.",
408 self.config.numEdgeSuspect, self.config.edgeMaskLevel)
410 # Depending on the value of config.matchExposuresType
411 # 'expTime' can stand for exposure time, flux, or ID.
412 for expTime in inputExp:
413 exposures = inputExp[expTime]
414 if len(exposures) == 1:
415 self.log.warning("Only one exposure found at %s %f. Dropping exposure %d.",
416 self.config.matchExposuresType, expTime, exposures[0][1])
417 continue
418 else:
419 # Only use the first two exposures at expTime. Each
420 # element is a tuple (exposure, expId)
421 expRef1, expId1 = exposures[0]
422 expRef2, expId2 = exposures[1]
423 # use get() to obtain `lsst.afw.image.Exposure`
424 exp1, exp2 = expRef1.get(), expRef2.get()
426 if len(exposures) > 2:
427 self.log.warning("Already found 2 exposures at %s %f. Ignoring exposures: %s",
428 self.config.matchExposuresType, expTime,
429 ", ".join(str(i[1]) for i in exposures[2:]))
430 # Mask pixels at the edge of the detector or of each amp
431 if self.config.numEdgeSuspect > 0:
432 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
433 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
434 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
435 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
437 nAmpsNan = 0
438 partialPtcDataset = PhotonTransferCurveDataset(ampNames, 'PARTIAL',
439 self.config.maximumRangeCovariancesAstier)
440 for ampNumber, amp in enumerate(detector):
441 ampName = amp.getName()
442 if self.config.detectorMeasurementRegion == 'AMP':
443 region = amp.getBBox()
444 elif self.config.detectorMeasurementRegion == 'FULL':
445 region = None
447 # Get masked image regions, masking planes, statistic control
448 # objects, and clipped means. Calculate once to reuse in
449 # `measureMeanVarCov` and `getGainFromFlatPair`.
450 im1Area, im2Area, imStatsCtrl, mu1, mu2 = self.getImageAreasMasksStats(exp1, exp2,
451 region=region)
453 # `measureMeanVarCov` is the function that measures
454 # the variance and covariances from a region of
455 # the difference image of two flats at the same
456 # exposure time. The variable `covAstier` that is
457 # returned is of the form:
458 # [(i, j, var (cov[0,0]), cov, npix) for (i,j) in
459 # {maxLag, maxLag}^2].
460 muDiff, varDiff, covAstier = self.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2)
461 # Estimate the gain from the flat pair
462 if self.config.doGain:
463 gain = self.getGainFromFlatPair(im1Area, im2Area, imStatsCtrl, mu1, mu2,
464 correctionType=self.config.gainCorrectionType,
465 readNoise=readNoiseDict[ampName])
466 else:
467 gain = np.nan
469 # Correction factor for bias introduced by sigma
470 # clipping.
471 # Function returns 1/sqrt(varFactor), so it needs
472 # to be squared. varDiff is calculated via
473 # afwMath.VARIANCECLIP.
474 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2
475 varDiff *= varFactor
477 expIdMask = True
478 # Mask data point at this mean signal level if
479 # the signal, variance, or covariance calculations
480 # from `measureMeanVarCov` resulted in NaNs.
481 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
482 self.log.warning("NaN mean or var, or None cov in amp %s in exposure pair %d, %d of "
483 "detector %d.", ampName, expId1, expId2, detNum)
484 nAmpsNan += 1
485 expIdMask = False
486 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
487 self.config.maximumRangeCovariancesAstier), np.nan)
488 covSqrtWeights = np.full_like(covArray, np.nan)
490 # Mask data point if it is outside of the
491 # specified mean signal range.
492 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
493 expIdMask = False
495 if covAstier is not None:
496 # Turn the tuples with the measured information
497 # into covariance arrays.
498 # covrow: (i, j, var (cov[0,0]), cov, npix)
499 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
500 ampName) for covRow in covAstier]
501 tempStructArray = np.array(tupleRows, dtype=tags)
503 covArray, vcov, _ = self.makeCovArray(tempStructArray,
504 self.config.maximumRangeCovariancesAstier)
506 # The returned covArray should only have 1 entry;
507 # raise if this is not the case.
508 if covArray.shape[0] != 1:
509 raise RuntimeError("Serious programming error in covArray shape.")
511 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
513 # Correct covArray for sigma clipping:
514 # 1) Apply varFactor twice for the whole covariance matrix
515 covArray *= varFactor**2
516 # 2) But, only once for the variance element of the
517 # matrix, covArray[0, 0, 0] (so divide one factor out).
518 # (the first 0 is because this is a 3D array for insertion into
519 # the combined dataset).
520 covArray[0, 0, 0] /= varFactor
522 if expIdMask:
523 # Run the Gaussian histogram only if this is a legal
524 # amplifier.
525 histVar, histChi2Dof, kspValue = self.computeGaussianHistogramParameters(
526 im1Area,
527 im2Area,
528 imStatsCtrl,
529 mu1,
530 mu2,
531 )
532 else:
533 histVar = np.nan
534 histChi2Dof = np.nan
535 kspValue = 0.0
537 partialPtcDataset.setAmpValuesPartialDataset(
538 ampName,
539 inputExpIdPair=(expId1, expId2),
540 rawExpTime=expTime,
541 rawMean=muDiff,
542 rawVar=varDiff,
543 expIdMask=expIdMask,
544 covariance=covArray[0, :, :],
545 covSqrtWeights=covSqrtWeights[0, :, :],
546 gain=gain,
547 noise=readNoiseDict[ampName],
548 histVar=histVar,
549 histChi2Dof=histChi2Dof,
550 kspValue=kspValue,
551 )
553 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
554 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple
555 # with a single-element array, so [0][0]
556 # is necessary to extract the required index.
557 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
558 # `partialPtcDatasetList` is a list of
559 # `PhotonTransferCurveDataset` objects. Some of them
560 # will be dummy datasets (to match length of input
561 # and output references), and the rest will have
562 # datasets with the mean signal, variance, and
563 # covariance measurements at a given exposure
564 # time. The next ppart of the PTC-measurement
565 # pipeline, `solve`, will take this list as input,
566 # and assemble the measurements in the datasets
567 # in an addecuate manner for fitting a PTC
568 # model.
569 partialPtcDataset.updateMetadataFromExposures([exp1, exp2])
570 partialPtcDataset.updateMetadata(setDate=True, detector=detector)
571 partialPtcDatasetList[datasetIndex] = partialPtcDataset
573 if nAmpsNan == len(ampNames):
574 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
575 self.log.warning(msg)
577 return pipeBase.Struct(
578 outputCovariances=partialPtcDatasetList,
579 )
581 def makeCovArray(self, inputTuple, maxRangeFromTuple):
582 """Make covariances array from tuple.
584 Parameters
585 ----------
586 inputTuple : `numpy.ndarray`
587 Structured array with rows with at least
588 (mu, afwVar, cov, var, i, j, npix), where:
589 mu : `float`
590 0.5*(m1 + m2), where mu1 is the mean value of flat1
591 and mu2 is the mean value of flat2.
592 afwVar : `float`
593 Variance of difference flat, calculated with afw.
594 cov : `float`
595 Covariance value at lag(i, j)
596 var : `float`
597 Variance(covariance value at lag(0, 0))
598 i : `int`
599 Lag in dimension "x".
600 j : `int`
601 Lag in dimension "y".
602 npix : `int`
603 Number of pixels used for covariance calculation.
604 maxRangeFromTuple : `int`
605 Maximum range to select from tuple.
607 Returns
608 -------
609 cov : `numpy.array`
610 Covariance arrays, indexed by mean signal mu.
611 vCov : `numpy.array`
612 Variance of the [co]variance arrays, indexed by mean signal mu.
613 muVals : `numpy.array`
614 List of mean signal values.
615 """
616 if maxRangeFromTuple is not None:
617 cut = (inputTuple['i'] < maxRangeFromTuple) & (inputTuple['j'] < maxRangeFromTuple)
618 cutTuple = inputTuple[cut]
619 else:
620 cutTuple = inputTuple
621 # increasing mu order, so that we can group measurements with the
622 # same mu
623 muTemp = cutTuple['mu']
624 ind = np.argsort(muTemp)
626 cutTuple = cutTuple[ind]
627 # should group measurements on the same image pairs(same average)
628 mu = cutTuple['mu']
629 xx = np.hstack(([mu[0]], mu))
630 delta = xx[1:] - xx[:-1]
631 steps, = np.where(delta > 0)
632 ind = np.zeros_like(mu, dtype=int)
633 ind[steps] = 1
634 ind = np.cumsum(ind) # this acts as an image pair index.
635 # now fill the 3-d cov array(and variance)
636 muVals = np.array(np.unique(mu))
637 i = cutTuple['i'].astype(int)
638 j = cutTuple['j'].astype(int)
639 c = 0.5*cutTuple['cov']
640 n = cutTuple['npix']
641 v = 0.5*cutTuple['var']
642 # book and fill
643 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1))
644 var = np.zeros_like(cov)
645 cov[ind, i, j] = c
646 var[ind, i, j] = v**2/n
647 var[:, 0, 0] *= 2 # var(v) = 2*v**2/N
649 return cov, var, muVals
651 def measureMeanVarCov(self, im1Area, im2Area, imStatsCtrl, mu1, mu2):
652 """Calculate the mean of each of two exposures and the variance
653 and covariance of their difference. The variance is calculated
654 via afwMath, and the covariance via the methods in Astier+19
655 (appendix A). In theory, var = covariance[0,0]. This should
656 be validated, and in the future, we may decide to just keep
657 one (covariance).
659 Parameters
660 ----------
661 im1Area : `lsst.afw.image.maskedImage.MaskedImageF`
662 Masked image from exposure 1.
663 im2Area : `lsst.afw.image.maskedImage.MaskedImageF`
664 Masked image from exposure 2.
665 imStatsCtrl : `lsst.afw.math.StatisticsControl`
666 Statistics control object.
667 mu1: `float`
668 Clipped mean of im1Area (ADU).
669 mu2: `float`
670 Clipped mean of im2Area (ADU).
672 Returns
673 -------
674 mu : `float` or `NaN`
675 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means
676 of the regions in both exposures. If either mu1 or m2 are
677 NaN's, the returned value is NaN.
678 varDiff : `float` or `NaN`
679 Half of the clipped variance of the difference of the
680 regions inthe two input exposures. If either mu1 or m2 are
681 NaN's, the returned value is NaN.
682 covDiffAstier : `list` or `NaN`
683 List with tuples of the form (dx, dy, var, cov, npix), where:
684 dx : `int`
685 Lag in x
686 dy : `int`
687 Lag in y
688 var : `float`
689 Variance at (dx, dy).
690 cov : `float`
691 Covariance at (dx, dy).
692 nPix : `int`
693 Number of pixel pairs used to evaluate var and cov.
695 If either mu1 or m2 are NaN's, the returned value is NaN.
696 """
697 if np.isnan(mu1) or np.isnan(mu2):
698 self.log.warning("Mean of amp in image 1 or 2 is NaN: %f, %f.", mu1, mu2)
699 return np.nan, np.nan, None
700 mu = 0.5*(mu1 + mu2)
702 # Take difference of pairs
703 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
704 temp = im2Area.clone()
705 temp *= mu1
706 diffIm = im1Area.clone()
707 diffIm *= mu2
708 diffIm -= temp
709 diffIm /= mu
711 if self.config.binSize > 1:
712 diffIm = afwMath.binImage(diffIm, self.config.binSize)
714 # Variance calculation via afwMath
715 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue())
717 # Covariances calculations
718 # Get the pixels that were not clipped
719 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue()
720 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, imStatsCtrl).getValue()
721 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
722 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
724 # Get the pixels in the mask planes of the difference image
725 # that were ignored by the clipping algorithm
726 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
727 # Combine the two sets of pixels ('1': use; '0': don't use)
728 # into a final weight matrix to be used in the covariance
729 # calculations below.
730 w = unmasked*wDiff
732 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2):
733 self.log.warning("Number of good points for covariance calculation (%s) is less "
734 "(than threshold %s)", np.sum(w),
735 self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2))
736 return np.nan, np.nan, None
738 maxRangeCov = self.config.maximumRangeCovariancesAstier
740 # Calculate covariances via FFT.
741 shapeDiff = np.array(diffIm.image.array.shape)
742 # Calculate the sizes of FFT dimensions.
743 s = shapeDiff + maxRangeCov
744 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
745 fftSize = np.array(2**(tempSize+1)).astype(int)
746 fftShape = (fftSize[0], fftSize[1])
747 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
748 # np.sum(w) is the same as npix[0][0] returned in covDiffAstier
749 try:
750 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
751 except ValueError:
752 # This is raised if there are not enough pixels.
753 self.log.warning("Not enough pixels covering the requested covariance range in x/y (%d)",
754 self.config.maximumRangeCovariancesAstier)
755 return np.nan, np.nan, None
757 # Compare Cov[0,0] and afwMath.VARIANCECLIP covDiffAstier[0]
758 # is the Cov[0,0] element, [3] is the variance, and there's a
759 # factor of 0.5 difference with afwMath.VARIANCECLIP.
760 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
761 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
762 if fractionalDiff >= thresholdPercentage:
763 self.log.warning("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
764 "is more than %f%%: %f", thresholdPercentage, fractionalDiff)
766 return mu, varDiff, covDiffAstier
768 def getImageAreasMasksStats(self, exposure1, exposure2, region=None):
769 """Get image areas in a region as well as masks and statistic objects.
771 Parameters
772 ----------
773 exposure1 : `lsst.afw.image.ExposureF`
774 First exposure of flat field pair.
775 exposure2 : `lsst.afw.image.ExposureF`
776 Second exposure of flat field pair.
777 region : `lsst.geom.Box2I`, optional
778 Region of each exposure where to perform the calculations
779 (e.g, an amplifier).
781 Returns
782 -------
783 im1Area : `lsst.afw.image.MaskedImageF`
784 Masked image from exposure 1.
785 im2Area : `lsst.afw.image.MaskedImageF`
786 Masked image from exposure 2.
787 imStatsCtrl : `lsst.afw.math.StatisticsControl`
788 Statistics control object.
789 mu1 : `float`
790 Clipped mean of im1Area (ADU).
791 mu2 : `float`
792 Clipped mean of im2Area (ADU).
793 """
794 if region is not None:
795 im1Area = exposure1.maskedImage[region]
796 im2Area = exposure2.maskedImage[region]
797 else:
798 im1Area = exposure1.maskedImage
799 im2Area = exposure2.maskedImage
801 # Get mask planes and construct statistics control object from one
802 # of the exposures
803 imMaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
804 imStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
805 self.config.nIterSigmaClipPtc,
806 imMaskVal)
807 imStatsCtrl.setNanSafe(True)
808 imStatsCtrl.setAndMask(imMaskVal)
810 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, imStatsCtrl).getValue()
811 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, imStatsCtrl).getValue()
813 return (im1Area, im2Area, imStatsCtrl, mu1, mu2)
815 def getGainFromFlatPair(self, im1Area, im2Area, imStatsCtrl, mu1, mu2,
816 correctionType='NONE', readNoise=None):
817 """Estimate the gain from a single pair of flats.
819 The basic premise is 1/g = <(I1 - I2)^2/(I1 + I2)> = 1/const,
820 where I1 and I2 correspond to flats 1 and 2, respectively.
821 Corrections for the variable QE and the read-noise are then
822 made following the derivation in Robert Lupton's forthcoming
823 book, which gets
825 1/g = <(I1 - I2)^2/(I1 + I2)> - 1/mu(sigma^2 - 1/2g^2).
827 This is a quadratic equation, whose solutions are given by:
829 g = mu +/- sqrt(2*sigma^2 - 2*const*mu + mu^2)/(2*const*mu*2
830 - 2*sigma^2)
832 where 'mu' is the average signal level and 'sigma' is the
833 amplifier's readnoise. The positive solution will be used.
834 The way the correction is applied depends on the value
835 supplied for correctionType.
837 correctionType is one of ['NONE', 'SIMPLE' or 'FULL']
838 'NONE' : uses the 1/g = <(I1 - I2)^2/(I1 + I2)> formula.
839 'SIMPLE' : uses the gain from the 'NONE' method for the
840 1/2g^2 term.
841 'FULL' : solves the full equation for g, discarding the
842 non-physical solution to the resulting quadratic.
844 Parameters
845 ----------
846 im1Area : `lsst.afw.image.maskedImage.MaskedImageF`
847 Masked image from exposure 1.
848 im2Area : `lsst.afw.image.maskedImage.MaskedImageF`
849 Masked image from exposure 2.
850 imStatsCtrl : `lsst.afw.math.StatisticsControl`
851 Statistics control object.
852 mu1: `float`
853 Clipped mean of im1Area (ADU).
854 mu2: `float`
855 Clipped mean of im2Area (ADU).
856 correctionType : `str`, optional
857 The correction applied, one of ['NONE', 'SIMPLE', 'FULL']
858 readNoise : `float`, optional
859 Amplifier readout noise (ADU).
861 Returns
862 -------
863 gain : `float`
864 Gain, in e/ADU.
866 Raises
867 ------
868 RuntimeError
869 Raise if `correctionType` is not one of 'NONE',
870 'SIMPLE', or 'FULL'.
871 """
872 if correctionType not in ['NONE', 'SIMPLE', 'FULL']:
873 raise RuntimeError("Unknown correction type: %s" % correctionType)
875 if correctionType != 'NONE' and not np.isfinite(readNoise):
876 self.log.warning("'correctionType' in 'getGainFromFlatPair' is %s, "
877 "but 'readNoise' is NaN. Setting 'correctionType' "
878 "to 'NONE', so a gain value will be estimated without "
879 "corrections." % correctionType)
880 correctionType = 'NONE'
882 mu = 0.5*(mu1 + mu2)
884 # ratioIm = (I1 - I2)^2 / (I1 + I2)
885 temp = im2Area.clone()
886 ratioIm = im1Area.clone()
887 ratioIm -= temp
888 ratioIm *= ratioIm
890 # Sum of pairs
891 sumIm = im1Area.clone()
892 sumIm += temp
894 ratioIm /= sumIm
896 const = afwMath.makeStatistics(ratioIm, afwMath.MEAN, imStatsCtrl).getValue()
897 gain = 1. / const
899 if correctionType == 'SIMPLE':
900 gain = 1/(const - (1/mu)*(readNoise**2 - (1/2*gain**2)))
901 elif correctionType == 'FULL':
902 root = np.sqrt(mu**2 - 2*mu*const + 2*readNoise**2)
903 denom = (2*const*mu - 2*readNoise**2)
904 positiveSolution = (root + mu)/denom
905 gain = positiveSolution
907 return gain
909 def getReadNoise(self, exposureMetadata, taskMetadata, ampName):
910 """Gets readout noise for an amp from ISR metadata.
912 If possible, this attempts to get the now-standard headers
913 added to the exposure itself. If not found there, the ISR
914 TaskMetadata is searched. If neither of these has the value,
915 warn and set the read noise to NaN.
917 Parameters
918 ----------
919 exposureMetadata : `lsst.daf.base.PropertySet`
920 Metadata to check for read noise first.
921 taskMetadata : `lsst.pipe.base.TaskMetadata`
922 List of exposures metadata from ISR for this exposure.
923 ampName : `str`
924 Amplifier name.
926 Returns
927 -------
928 readNoise : `float`
929 The read noise for this set of exposure/amplifier.
930 """
931 # Try from the exposure first.
932 expectedKey = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {ampName}"
933 if expectedKey in exposureMetadata:
934 return exposureMetadata[expectedKey]
936 # If not, try getting it from the task metadata.
937 expectedKey = f"RESIDUAL STDEV {ampName}"
938 if "isr" in taskMetadata:
939 if expectedKey in taskMetadata["isr"]:
940 return taskMetadata["isr"][expectedKey]
942 self.log.warning("Median readout noise from ISR metadata for amp %s "
943 "could not be calculated." % ampName)
944 return np.nan
946 def computeGaussianHistogramParameters(self, im1Area, im2Area, imStatsCtrl, mu1, mu2):
947 """Compute KS test for a Gaussian model fit to a histogram of the
948 difference image.
950 Parameters
951 ----------
952 im1Area : `lsst.afw.image.MaskedImageF`
953 Masked image from exposure 1.
954 im2Area : `lsst.afw.image.MaskedImageF`
955 Masked image from exposure 2.
956 imStatsCtrl : `lsst.afw.math.StatisticsControl`
957 Statistics control object.
958 mu1 : `float`
959 Clipped mean of im1Area (ADU).
960 mu2 : `float`
961 Clipped mean of im2Area (ADU).
963 Returns
964 -------
965 varFit : `float`
966 Variance from the Gaussian fit.
967 chi2Dof : `float`
968 Chi-squared per degree of freedom of Gaussian fit.
969 kspValue : `float`
970 The KS test p-value for the Gaussian fit.
972 Notes
973 -----
974 The algorithm here was originally developed by Aaron Roodman.
975 Tests on the full focal plane of LSSTCam during testing has shown
976 that a KS test p-value cut of 0.01 is a good discriminant for
977 well-behaved flat pairs (p>0.01) and poorly behaved non-Gaussian
978 flat pairs (p<0.01).
979 """
980 diffExp = im1Area.clone()
981 diffExp -= im2Area
983 sel = (((diffExp.mask.array & imStatsCtrl.getAndMask()) == 0)
984 & np.isfinite(diffExp.mask.array))
985 diffArr = diffExp.image.array[sel]
987 numOk = len(diffArr)
989 if numOk >= self.config.ksHistMinDataValues and np.isfinite(mu1) and np.isfinite(mu2):
990 # Create a histogram symmetric around zero, with a bin size
991 # determined from the expected variance given by the average of
992 # the input signal levels.
993 lim = self.config.ksHistLimitMultiplier * np.sqrt((mu1 + mu2)/2.)
994 yVals, binEdges = np.histogram(diffArr, bins=self.config.ksHistNBins, range=[-lim, lim])
996 # Fit the histogram with a Gaussian model.
997 model = GaussianModel()
998 yVals = yVals.astype(np.float64)
999 xVals = ((binEdges[0: -1] + binEdges[1:])/2.).astype(np.float64)
1000 errVals = np.sqrt(yVals)
1001 errVals[(errVals == 0.0)] = 1.0
1002 pars = model.guess(yVals, x=xVals)
1003 out = model.fit(yVals, pars, x=xVals, weights=1./errVals, calc_covar=True, method="least_squares")
1005 # Calculate chi2.
1006 chiArr = out.residual
1007 nDof = len(yVals) - 3
1008 chi2Dof = np.sum(chiArr**2.)/nDof
1009 sigmaFit = out.params["sigma"].value
1011 # Calculate KS test p-value for the fit.
1012 # Seed this with the mean value, so that the same data will get the
1013 # same result.
1014 randomSeed = int((mu1 + mu2)/2.)
1015 gSample = scipy.stats.norm.rvs(
1016 size=numOk,
1017 scale=sigmaFit,
1018 loc=out.params["center"].value,
1019 random_state=randomSeed,
1020 )
1021 ksResult = scipy.stats.ks_2samp(diffArr, gSample)
1022 kspValue = ksResult.pvalue
1023 if kspValue < 1e-15:
1024 kspValue = 0.0
1026 varFit = sigmaFit**2.
1028 else:
1029 varFit = np.nan
1030 chi2Dof = np.nan
1031 kspValue = 0.0
1033 return varFit, chi2Dof, kspValue