Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py: 11%
356 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-01-31 12:22 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-01-31 12:22 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
23from lmfit.models import GaussianModel
24import scipy.stats
25import warnings
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30from lsst.geom import (Box2I, Point2I, Extent2I)
31from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId,
32 arrangeFlatsByExpFlux, sigmaClipCorrection,
33 CovFastFourierTransform)
35import lsst.pipe.base.connectionTypes as cT
37from lsst.ip.isr import PhotonTransferCurveDataset
38from lsst.ip.isr import IsrTask
40__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask']
43class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections,
44 dimensions=("instrument", "detector")):
46 inputExp = cT.Input(
47 name="ptcInputExposurePairs",
48 doc="Input post-ISR processed exposure pairs (flats) to"
49 "measure covariances from.",
50 storageClass="Exposure",
51 dimensions=("instrument", "exposure", "detector"),
52 multiple=True,
53 deferLoad=True,
54 )
55 inputPhotodiodeData = cT.Input(
56 name="photodiode",
57 doc="Photodiode readings data.",
58 storageClass="IsrCalib",
59 dimensions=("instrument", "exposure"),
60 multiple=True,
61 deferLoad=True,
62 )
63 taskMetadata = cT.Input(
64 name="isr_metadata",
65 doc="Input task metadata to extract statistics from.",
66 storageClass="TaskMetadata",
67 dimensions=("instrument", "exposure", "detector"),
68 multiple=True,
69 )
70 outputCovariances = cT.Output(
71 name="ptcCovariances",
72 doc="Extracted flat (co)variances.",
73 storageClass="PhotonTransferCurveDataset",
74 dimensions=("instrument", "exposure", "detector"),
75 isCalibration=True,
76 multiple=True,
77 )
79 def __init__(self, *, config=None):
80 if not config.doExtractPhotodiodeData:
81 del self.inputPhotodiodeData
84class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=PhotonTransferCurveExtractConnections):
86 """Configuration for the measurement of covariances from flats.
87 """
88 matchExposuresType = pexConfig.ChoiceField(
89 dtype=str,
90 doc="Match input exposures by time, flux, or expId",
91 default='TIME',
92 allowed={
93 "TIME": "Match exposures by exposure time.",
94 "FLUX": "Match exposures by target flux. Use header keyword"
95 " in matchExposuresByFluxKeyword to find the flux.",
96 "EXPID": "Match exposures by exposure ID."
97 }
98 )
99 matchExposuresByFluxKeyword = pexConfig.Field(
100 dtype=str,
101 doc="Header keyword for flux if matchExposuresType is FLUX.",
102 default='CCOBFLUX',
103 )
104 maximumRangeCovariancesAstier = pexConfig.Field(
105 dtype=int,
106 doc="Maximum range of covariances as in Astier+19",
107 default=8,
108 )
109 binSize = pexConfig.Field(
110 dtype=int,
111 doc="Bin the image by this factor in both dimensions.",
112 default=1,
113 )
114 minMeanSignal = pexConfig.DictField(
115 keytype=str,
116 itemtype=float,
117 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use."
118 " The same cut is applied to all amps if this parameter [`dict`] is passed as "
119 " {'ALL_AMPS': value}",
120 default={'ALL_AMPS': 0.0},
121 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.",
122 )
123 maxMeanSignal = pexConfig.DictField(
124 keytype=str,
125 itemtype=float,
126 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp."
127 " The same cut is applied to all amps if this dictionary is of the form"
128 " {'ALL_AMPS': value}",
129 default={'ALL_AMPS': 1e6},
130 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.",
131 )
132 maskNameList = pexConfig.ListField(
133 dtype=str,
134 doc="Mask list to exclude from statistics calculations.",
135 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'],
136 )
137 nSigmaClipPtc = pexConfig.Field(
138 dtype=float,
139 doc="Sigma cut for afwMath.StatisticsControl()",
140 default=5.5,
141 )
142 nIterSigmaClipPtc = pexConfig.Field(
143 dtype=int,
144 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()",
145 default=3,
146 )
147 minNumberGoodPixelsForCovariance = pexConfig.Field(
148 dtype=int,
149 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or"
150 " direclty).",
151 default=10000,
152 )
153 thresholdDiffAfwVarVsCov00 = pexConfig.Field(
154 dtype=float,
155 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 "
156 "for a region of a difference image is greater than this threshold (percentage), "
157 "a warning will be issued.",
158 default=1.,
159 )
160 detectorMeasurementRegion = pexConfig.ChoiceField(
161 dtype=str,
162 doc="Region of each exposure where to perform the calculations (amplifier or full image).",
163 default='AMP',
164 allowed={
165 "AMP": "Amplifier of the detector.",
166 "FULL": "Full image."
167 }
168 )
169 numEdgeSuspect = pexConfig.Field(
170 dtype=int,
171 doc="Number of edge pixels to be flagged as untrustworthy.",
172 default=0,
173 )
174 edgeMaskLevel = pexConfig.ChoiceField(
175 dtype=str,
176 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
177 default="DETECTOR",
178 allowed={
179 'DETECTOR': 'Mask only the edges of the full detector.',
180 'AMP': 'Mask edges of each amplifier.',
181 },
182 )
183 doGain = pexConfig.Field(
184 dtype=bool,
185 doc="Calculate a gain per input flat pair.",
186 default=True,
187 )
188 gainCorrectionType = pexConfig.ChoiceField(
189 dtype=str,
190 doc="Correction type for the gain.",
191 default='FULL',
192 allowed={
193 'NONE': 'No correction.',
194 'SIMPLE': 'First order correction.',
195 'FULL': 'Second order correction.'
196 }
197 )
198 ksHistNBins = pexConfig.Field(
199 dtype=int,
200 doc="Number of bins for the KS test histogram.",
201 default=100,
202 )
203 ksHistLimitMultiplier = pexConfig.Field(
204 dtype=float,
205 doc="Number of sigma (as predicted from the mean value) to compute KS test histogram.",
206 default=8.0,
207 )
208 ksHistMinDataValues = pexConfig.Field(
209 dtype=int,
210 doc="Minimum number of good data values to compute KS test histogram.",
211 default=100,
212 )
213 auxiliaryHeaderKeys = pexConfig.ListField(
214 dtype=str,
215 doc="Auxiliary header keys to store with the PTC dataset.",
216 default=[],
217 )
218 doExtractPhotodiodeData = pexConfig.Field(
219 dtype=bool,
220 doc="Extract photodiode data?",
221 default=False,
222 )
223 photodiodeIntegrationMethod = pexConfig.ChoiceField(
224 dtype=str,
225 doc="Integration method for photodiode monitoring data.",
226 default="CHARGE_SUM",
227 allowed={
228 "DIRECT_SUM": ("Use numpy's trapz integrator on all photodiode "
229 "readout entries"),
230 "TRIMMED_SUM": ("Use numpy's trapz integrator, clipping the "
231 "leading and trailing entries, which are "
232 "nominally at zero baseline level."),
233 "CHARGE_SUM": ("Treat the current values as integrated charge "
234 "over the sampling interval and simply sum "
235 "the values, after subtracting a baseline level."),
236 },
237 )
238 photodiodeCurrentScale = pexConfig.Field(
239 dtype=float,
240 doc="Scale factor to apply to photodiode current values for the "
241 "``CHARGE_SUM`` integration method.",
242 default=-1.0,
243 )
246class PhotonTransferCurveExtractTask(pipeBase.PipelineTask):
247 """Task to measure covariances from flat fields.
249 This task receives as input a list of flat-field images
250 (flats), and sorts these flats in pairs taken at the
251 same time (the task will raise if there is one one flat
252 at a given exposure time, and it will discard extra flats if
253 there are more than two per exposure time). This task measures
254 the mean, variance, and covariances from a region (e.g.,
255 an amplifier) of the difference image of the two flats with
256 the same exposure time (alternatively, all input images could have
257 the same exposure time but their flux changed).
259 The variance is calculated via afwMath, and the covariance
260 via the methods in Astier+19 (appendix A). In theory,
261 var = covariance[0,0]. This should be validated, and in the
262 future, we may decide to just keep one (covariance).
263 At this moment, if the two values differ by more than the value
264 of `thresholdDiffAfwVarVsCov00` (default: 1%), a warning will
265 be issued.
267 The measured covariances at a given exposure time (along with
268 other quantities such as the mean) are stored in a PTC dataset
269 object (`~lsst.ip.isr.PhotonTransferCurveDataset`), which gets
270 partially filled at this stage (the remainder of the attributes
271 of the dataset will be filled after running the second task of
272 the PTC-measurement pipeline, `~PhotonTransferCurveSolveTask`).
274 The number of partially-filled
275 `~lsst.ip.isr.PhotonTransferCurveDataset` objects will be less
276 than the number of input exposures because the task combines
277 input flats in pairs. However, it is required at this moment
278 that the number of input dimensions matches
279 bijectively the number of output dimensions. Therefore, a number
280 of "dummy" PTC datasets are inserted in the output list. This
281 output list will then be used as input of the next task in the
282 PTC-measurement pipeline, `PhotonTransferCurveSolveTask`,
283 which will assemble the multiple `PhotonTransferCurveDataset`
284 objects into a single one in order to fit the measured covariances
285 as a function of flux to one of three models
286 (see `PhotonTransferCurveSolveTask` for details).
288 Reference: Astier+19: "The Shape of the Photon Transfer Curve of CCD
289 sensors", arXiv:1905.08677.
290 """
292 ConfigClass = PhotonTransferCurveExtractConfig
293 _DefaultName = 'cpPtcExtract'
295 def runQuantum(self, butlerQC, inputRefs, outputRefs):
296 """Ensure that the input and output dimensions are passed along.
298 Parameters
299 ----------
300 butlerQC : `~lsst.daf.butler.QuantumContext`
301 Butler to operate on.
302 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`
303 Input data refs to load.
304 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection`
305 Output data refs to persist.
306 """
307 inputs = butlerQC.get(inputRefs)
308 # Ids of input list of exposure references
309 # (deferLoad=True in the input connections)
310 inputs['inputDims'] = [expRef.datasetRef.dataId['exposure'] for expRef in inputRefs.inputExp]
312 # Dictionary, keyed by expTime (or expFlux or expId), with tuples
313 # containing flat exposures and their IDs.
314 matchType = self.config.matchExposuresType
315 if matchType == 'TIME':
316 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims'], log=self.log)
317 elif matchType == 'FLUX':
318 inputs['inputExp'] = arrangeFlatsByExpFlux(
319 inputs['inputExp'],
320 inputs['inputDims'],
321 self.config.matchExposuresByFluxKeyword,
322 log=self.log,
323 )
324 else:
325 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims'])
327 outputs = self.run(**inputs)
328 outputs = self._guaranteeOutputs(inputs['inputDims'], outputs, outputRefs)
329 butlerQC.put(outputs, outputRefs)
331 def _guaranteeOutputs(self, inputDims, outputs, outputRefs):
332 """Ensure that all outputRefs have a matching output, and if they do
333 not, fill the output with dummy PTC datasets.
335 Parameters
336 ----------
337 inputDims : `dict` [`str`, `int`]
338 Input exposure dimensions.
339 outputs : `lsst.pipe.base.Struct`
340 Outputs from the ``run`` method. Contains the entry:
342 ``outputCovariances``
343 Output PTC datasets (`list` [`lsst.ip.isr.IsrCalib`])
344 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection`
345 Container with all of the outputs expected to be generated.
347 Returns
348 -------
349 outputs : `lsst.pipe.base.Struct`
350 Dummy dataset padded version of the input ``outputs`` with
351 the same entries.
352 """
353 newCovariances = []
354 for ref in outputRefs.outputCovariances:
355 outputExpId = ref.dataId['exposure']
356 if outputExpId in inputDims:
357 entry = inputDims.index(outputExpId)
358 newCovariances.append(outputs.outputCovariances[entry])
359 else:
360 newPtc = PhotonTransferCurveDataset(['no amp'], 'DUMMY', covMatrixSide=1)
361 newPtc.setAmpValuesPartialDataset('no amp')
362 newCovariances.append(newPtc)
363 return pipeBase.Struct(outputCovariances=newCovariances)
365 def run(self, inputExp, inputDims, taskMetadata, inputPhotodiodeData=None):
367 """Measure covariances from difference of flat pairs
369 Parameters
370 ----------
371 inputExp : `dict` [`float`, `list`
372 [`~lsst.pipe.base.connections.DeferredDatasetRef`]]
373 Dictionary that groups references to flat-field exposures that
374 have the same exposure time (seconds), or that groups them
375 sequentially by their exposure id.
376 inputDims : `list`
377 List of exposure IDs.
378 taskMetadata : `list` [`lsst.pipe.base.TaskMetadata`]
379 List of exposures metadata from ISR.
380 inputPhotodiodeData : `dict` [`str`, `lsst.ip.isr.PhotodiodeCalib`]
381 Photodiode readings data (optional).
383 Returns
384 -------
385 results : `lsst.pipe.base.Struct`
386 The resulting Struct contains:
388 ``outputCovariances``
389 A list containing the per-pair PTC measurements (`list`
390 [`lsst.ip.isr.PhotonTransferCurveDataset`])
391 """
392 # inputExp.values() returns a view, which we turn into a list. We then
393 # access the first exposure-ID tuple to get the detector.
394 # The first "get()" retrieves the exposure from the exposure reference.
395 detector = list(inputExp.values())[0][0][0].get(component='detector')
396 detNum = detector.getId()
397 amps = detector.getAmplifiers()
398 ampNames = [amp.getName() for amp in amps]
400 # Each amp may have a different min and max ADU signal
401 # specified in the config.
402 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
403 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
404 for ampName in ampNames:
405 if 'ALL_AMPS' in self.config.maxMeanSignal:
406 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
407 elif ampName in self.config.maxMeanSignal:
408 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
410 if 'ALL_AMPS' in self.config.minMeanSignal:
411 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
412 elif ampName in self.config.minMeanSignal:
413 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
414 # These are the column names for `tupleRows` below.
415 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
416 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
417 # Create a dummy ptcDataset. Dummy datasets will be
418 # used to ensure that the number of output and input
419 # dimensions match.
420 dummyPtcDataset = PhotonTransferCurveDataset(
421 ampNames, 'DUMMY',
422 covMatrixSide=self.config.maximumRangeCovariancesAstier)
423 for ampName in ampNames:
424 dummyPtcDataset.setAmpValuesPartialDataset(ampName)
426 # Extract the photodiode data if requested.
427 if self.config.doExtractPhotodiodeData:
428 # Compute the photodiode integrals once, at the start.
429 monitorDiodeCharge = {}
430 for handle in inputPhotodiodeData:
431 expId = handle.dataId['exposure']
432 pdCalib = handle.get()
433 pdCalib.integrationMethod = self.config.photodiodeIntegrationMethod
434 pdCalib.currentScale = self.config.photodiodeCurrentScale
435 monitorDiodeCharge[expId] = pdCalib.integrate()
437 # Get read noise. Try from the exposure, then try
438 # taskMetadata. This adds a get() for the exposures.
439 readNoiseLists = {}
440 for pairIndex, expRefs in inputExp.items():
441 # This yields an index (exposure_time, seq_num, or flux)
442 # and a pair of references at that index.
443 for expRef, expId in expRefs:
444 # This yields an exposure ref and an exposureId.
445 exposureMetadata = expRef.get(component="metadata")
446 metadataIndex = inputDims.index(expId)
447 thisTaskMetadata = taskMetadata[metadataIndex]
449 for ampName in ampNames:
450 if ampName not in readNoiseLists:
451 readNoiseLists[ampName] = [self.getReadNoise(exposureMetadata,
452 thisTaskMetadata, ampName)]
453 else:
454 readNoiseLists[ampName].append(self.getReadNoise(exposureMetadata,
455 thisTaskMetadata, ampName))
457 readNoiseDict = {ampName: 0.0 for ampName in ampNames}
458 for ampName in ampNames:
459 # Take median read noise value
460 readNoiseDict[ampName] = np.nanmedian(readNoiseLists[ampName])
462 # Output list with PTC datasets.
463 partialPtcDatasetList = []
464 # The number of output references needs to match that of input
465 # references: initialize outputlist with dummy PTC datasets.
466 for i in range(len(inputDims)):
467 partialPtcDatasetList.append(dummyPtcDataset)
469 if self.config.numEdgeSuspect > 0:
470 isrTask = IsrTask()
471 self.log.info("Masking %d pixels from the edges of all %ss as SUSPECT.",
472 self.config.numEdgeSuspect, self.config.edgeMaskLevel)
474 # Depending on the value of config.matchExposuresType
475 # 'expTime' can stand for exposure time, flux, or ID.
476 for expTime in inputExp:
477 exposures = inputExp[expTime]
478 if not np.isfinite(expTime):
479 self.log.warning("Illegal/missing %s found (%s). Dropping exposure %d",
480 self.config.matchExposuresType, expTime, exposures[0][1])
481 continue
482 elif len(exposures) == 1:
483 self.log.warning("Only one exposure found at %s %f. Dropping exposure %d.",
484 self.config.matchExposuresType, expTime, exposures[0][1])
485 continue
486 else:
487 # Only use the first two exposures at expTime. Each
488 # element is a tuple (exposure, expId)
489 expRef1, expId1 = exposures[0]
490 expRef2, expId2 = exposures[1]
491 # use get() to obtain `lsst.afw.image.Exposure`
492 exp1, exp2 = expRef1.get(), expRef2.get()
494 if len(exposures) > 2:
495 self.log.warning("Already found 2 exposures at %s %f. Ignoring exposures: %s",
496 self.config.matchExposuresType, expTime,
497 ", ".join(str(i[1]) for i in exposures[2:]))
498 # Mask pixels at the edge of the detector or of each amp
499 if self.config.numEdgeSuspect > 0:
500 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect,
501 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
502 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect,
503 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
505 # Extract any metadata keys from the headers.
506 auxDict = {}
507 metadata = exp1.getMetadata()
508 for key in self.config.auxiliaryHeaderKeys:
509 if key not in metadata:
510 self.log.warning(
511 "Requested auxiliary keyword %s not found in exposure metadata for %d",
512 key,
513 expId1,
514 )
515 value = np.nan
516 else:
517 value = metadata[key]
519 auxDict[key] = value
521 nAmpsNan = 0
522 partialPtcDataset = PhotonTransferCurveDataset(
523 ampNames, 'PARTIAL',
524 covMatrixSide=self.config.maximumRangeCovariancesAstier)
525 for ampNumber, amp in enumerate(detector):
526 ampName = amp.getName()
527 if self.config.detectorMeasurementRegion == 'AMP':
528 region = amp.getBBox()
529 elif self.config.detectorMeasurementRegion == 'FULL':
530 region = None
532 # Get masked image regions, masking planes, statistic control
533 # objects, and clipped means. Calculate once to reuse in
534 # `measureMeanVarCov` and `getGainFromFlatPair`.
535 im1Area, im2Area, imStatsCtrl, mu1, mu2 = self.getImageAreasMasksStats(exp1, exp2,
536 region=region)
538 # We demand that both mu1 and mu2 be finite and greater than 0.
539 if not np.isfinite(mu1) or not np.isfinite(mu2) \
540 or ((np.nan_to_num(mu1) + np.nan_to_num(mu2)/2.) <= 0.0):
541 self.log.warning(
542 "Illegal mean value(s) detected for amp %s on exposure pair %d/%d",
543 ampName,
544 expId1,
545 expId2,
546 )
547 partialPtcDataset.setAmpValuesPartialDataset(
548 ampName,
549 inputExpIdPair=(expId1, expId2),
550 rawExpTime=expTime,
551 expIdMask=False,
552 )
553 continue
555 # `measureMeanVarCov` is the function that measures
556 # the variance and covariances from a region of
557 # the difference image of two flats at the same
558 # exposure time. The variable `covAstier` that is
559 # returned is of the form:
560 # [(i, j, var (cov[0,0]), cov, npix) for (i,j) in
561 # {maxLag, maxLag}^2].
562 muDiff, varDiff, covAstier, rowMeanVariance = self.measureMeanVarCov(im1Area,
563 im2Area,
564 imStatsCtrl,
565 mu1,
566 mu2)
567 # Estimate the gain from the flat pair
568 if self.config.doGain:
569 gain = self.getGainFromFlatPair(im1Area, im2Area, imStatsCtrl, mu1, mu2,
570 correctionType=self.config.gainCorrectionType,
571 readNoise=readNoiseDict[ampName])
572 else:
573 gain = np.nan
575 # Correction factor for bias introduced by sigma
576 # clipping.
577 # Function returns 1/sqrt(varFactor), so it needs
578 # to be squared. varDiff is calculated via
579 # afwMath.VARIANCECLIP.
580 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2
581 varDiff *= varFactor
583 expIdMask = True
584 # Mask data point at this mean signal level if
585 # the signal, variance, or covariance calculations
586 # from `measureMeanVarCov` resulted in NaNs.
587 if (np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None)
588 or (rowMeanVariance is np.nan)):
589 self.log.warning("NaN mean, var or rowmeanVariance, or None cov in amp %s "
590 "in exposure pair %d, %d of "
591 "detector %d.", ampName, expId1, expId2, detNum)
592 nAmpsNan += 1
593 expIdMask = False
594 covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
595 self.config.maximumRangeCovariancesAstier), np.nan)
596 covSqrtWeights = np.full_like(covArray, np.nan)
598 # Mask data point if it is outside of the
599 # specified mean signal range.
600 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
601 expIdMask = False
603 if covAstier is not None:
604 # Turn the tuples with the measured information
605 # into covariance arrays.
606 # covrow: (i, j, var (cov[0,0]), cov, npix)
607 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
608 ampName) for covRow in covAstier]
609 tempStructArray = np.array(tupleRows, dtype=tags)
611 covArray, vcov, _ = self.makeCovArray(tempStructArray,
612 self.config.maximumRangeCovariancesAstier)
614 # The returned covArray should only have 1 entry;
615 # raise if this is not the case.
616 if covArray.shape[0] != 1:
617 raise RuntimeError("Serious programming error in covArray shape.")
619 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
621 # Correct covArray for sigma clipping:
622 # 1) Apply varFactor twice for the whole covariance matrix
623 covArray *= varFactor**2
624 # 2) But, only once for the variance element of the
625 # matrix, covArray[0, 0, 0] (so divide one factor out).
626 # (the first 0 is because this is a 3D array for insertion into
627 # the combined dataset).
628 covArray[0, 0, 0] /= varFactor
630 if expIdMask:
631 # Run the Gaussian histogram only if this is a legal
632 # amplifier.
633 histVar, histChi2Dof, kspValue = self.computeGaussianHistogramParameters(
634 im1Area,
635 im2Area,
636 imStatsCtrl,
637 mu1,
638 mu2,
639 )
640 else:
641 histVar = np.nan
642 histChi2Dof = np.nan
643 kspValue = 0.0
645 if self.config.doExtractPhotodiodeData:
646 nExps = 0
647 photoCharge = 0.0
648 for expId in [expId1, expId2]:
649 if expId in monitorDiodeCharge:
650 photoCharge += monitorDiodeCharge[expId]
651 nExps += 1
652 if nExps > 0:
653 photoCharge /= nExps
654 else:
655 photoCharge = np.nan
656 else:
657 photoCharge = np.nan
659 partialPtcDataset.setAmpValuesPartialDataset(
660 ampName,
661 inputExpIdPair=(expId1, expId2),
662 rawExpTime=expTime,
663 rawMean=muDiff,
664 rawVar=varDiff,
665 photoCharge=photoCharge,
666 expIdMask=expIdMask,
667 covariance=covArray[0, :, :],
668 covSqrtWeights=covSqrtWeights[0, :, :],
669 gain=gain,
670 noise=readNoiseDict[ampName],
671 histVar=histVar,
672 histChi2Dof=histChi2Dof,
673 kspValue=kspValue,
674 rowMeanVariance=rowMeanVariance,
675 )
677 partialPtcDataset.setAuxValuesPartialDataset(auxDict)
679 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
680 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple
681 # with a single-element array, so [0][0]
682 # is necessary to extract the required index.
683 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
684 # `partialPtcDatasetList` is a list of
685 # `PhotonTransferCurveDataset` objects. Some of them
686 # will be dummy datasets (to match length of input
687 # and output references), and the rest will have
688 # datasets with the mean signal, variance, and
689 # covariance measurements at a given exposure
690 # time. The next ppart of the PTC-measurement
691 # pipeline, `solve`, will take this list as input,
692 # and assemble the measurements in the datasets
693 # in an addecuate manner for fitting a PTC
694 # model.
695 partialPtcDataset.updateMetadataFromExposures([exp1, exp2])
696 partialPtcDataset.updateMetadata(setDate=True, detector=detector)
697 partialPtcDatasetList[datasetIndex] = partialPtcDataset
699 if nAmpsNan == len(ampNames):
700 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
701 self.log.warning(msg)
703 return pipeBase.Struct(
704 outputCovariances=partialPtcDatasetList,
705 )
707 def makeCovArray(self, inputTuple, maxRangeFromTuple):
708 """Make covariances array from tuple.
710 Parameters
711 ----------
712 inputTuple : `numpy.ndarray`
713 Structured array with rows with at least
714 (mu, afwVar, cov, var, i, j, npix), where:
715 mu : `float`
716 0.5*(m1 + m2), where mu1 is the mean value of flat1
717 and mu2 is the mean value of flat2.
718 afwVar : `float`
719 Variance of difference flat, calculated with afw.
720 cov : `float`
721 Covariance value at lag(i, j)
722 var : `float`
723 Variance(covariance value at lag(0, 0))
724 i : `int`
725 Lag in dimension "x".
726 j : `int`
727 Lag in dimension "y".
728 npix : `int`
729 Number of pixels used for covariance calculation.
730 maxRangeFromTuple : `int`
731 Maximum range to select from tuple.
733 Returns
734 -------
735 cov : `numpy.array`
736 Covariance arrays, indexed by mean signal mu.
737 vCov : `numpy.array`
738 Variance of the [co]variance arrays, indexed by mean signal mu.
739 muVals : `numpy.array`
740 List of mean signal values.
741 """
742 if maxRangeFromTuple is not None:
743 cut = (inputTuple['i'] < maxRangeFromTuple) & (inputTuple['j'] < maxRangeFromTuple)
744 cutTuple = inputTuple[cut]
745 else:
746 cutTuple = inputTuple
747 # increasing mu order, so that we can group measurements with the
748 # same mu
749 muTemp = cutTuple['mu']
750 ind = np.argsort(muTemp)
752 cutTuple = cutTuple[ind]
753 # should group measurements on the same image pairs(same average)
754 mu = cutTuple['mu']
755 xx = np.hstack(([mu[0]], mu))
756 delta = xx[1:] - xx[:-1]
757 steps, = np.where(delta > 0)
758 ind = np.zeros_like(mu, dtype=int)
759 ind[steps] = 1
760 ind = np.cumsum(ind) # this acts as an image pair index.
761 # now fill the 3-d cov array(and variance)
762 muVals = np.array(np.unique(mu))
763 i = cutTuple['i'].astype(int)
764 j = cutTuple['j'].astype(int)
765 c = 0.5*cutTuple['cov']
766 n = cutTuple['npix']
767 v = 0.5*cutTuple['var']
768 # book and fill
769 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1))
770 var = np.zeros_like(cov)
771 cov[ind, i, j] = c
772 var[ind, i, j] = v**2/n
773 var[:, 0, 0] *= 2 # var(v) = 2*v**2/N
775 return cov, var, muVals
777 def measureMeanVarCov(self, im1Area, im2Area, imStatsCtrl, mu1, mu2):
778 """Calculate the mean of each of two exposures and the variance
779 and covariance of their difference. The variance is calculated
780 via afwMath, and the covariance via the methods in Astier+19
781 (appendix A). In theory, var = covariance[0,0]. This should
782 be validated, and in the future, we may decide to just keep
783 one (covariance).
785 Parameters
786 ----------
787 im1Area : `lsst.afw.image.maskedImage.MaskedImageF`
788 Masked image from exposure 1.
789 im2Area : `lsst.afw.image.maskedImage.MaskedImageF`
790 Masked image from exposure 2.
791 imStatsCtrl : `lsst.afw.math.StatisticsControl`
792 Statistics control object.
793 mu1: `float`
794 Clipped mean of im1Area (ADU).
795 mu2: `float`
796 Clipped mean of im2Area (ADU).
798 Returns
799 -------
800 mu : `float`
801 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means
802 of the regions in both exposures. If either mu1 or m2 are
803 NaN's, the returned value is NaN.
804 varDiff : `float`
805 Half of the clipped variance of the difference of the
806 regions inthe two input exposures. If either mu1 or m2 are
807 NaN's, the returned value is NaN.
808 covDiffAstier : `list` or `NaN`
809 List with tuples of the form (dx, dy, var, cov, npix), where:
810 dx : `int`
811 Lag in x
812 dy : `int`
813 Lag in y
814 var : `float`
815 Variance at (dx, dy).
816 cov : `float`
817 Covariance at (dx, dy).
818 nPix : `int`
819 Number of pixel pairs used to evaluate var and cov.
820 rowMeanVariance : `float`
821 Variance of the means of each row in the difference image.
822 Taken from `github.com/lsst-camera-dh/eo_pipe`.
824 If either mu1 or m2 are NaN's, the returned value is NaN.
825 """
826 if np.isnan(mu1) or np.isnan(mu2):
827 self.log.warning("Mean of amp in image 1 or 2 is NaN: %f, %f.", mu1, mu2)
828 return np.nan, np.nan, None, np.nan
829 mu = 0.5*(mu1 + mu2)
831 # Take difference of pairs
832 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
833 temp = im2Area.clone()
834 temp *= mu1
835 diffIm = im1Area.clone()
836 diffIm *= mu2
837 diffIm -= temp
838 diffIm /= mu
840 if self.config.binSize > 1:
841 diffIm = afwMath.binImage(diffIm, self.config.binSize)
843 # Calculate the variance (in ADU^2) of the means of rows for diffIm.
844 # Taken from eo_pipe
845 region = diffIm.getBBox()
846 rowMeans = []
847 for row in range(region.minY, region.maxY):
848 regionRow = Box2I(Point2I(region.minX, row),
849 Extent2I(region.width, 1))
850 rowMeans.append(afwMath.makeStatistics(diffIm[regionRow],
851 afwMath.MEAN,
852 imStatsCtrl).getValue())
853 rowMeanVariance = afwMath.makeStatistics(
854 np.array(rowMeans), afwMath.VARIANCECLIP,
855 imStatsCtrl).getValue()
857 # Variance calculation via afwMath
858 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue())
860 # Covariances calculations
861 # Get the pixels that were not clipped
862 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue()
863 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, imStatsCtrl).getValue()
864 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip)
865 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0)
867 # Get the pixels in the mask planes of the difference image
868 # that were ignored by the clipping algorithm
869 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
870 # Combine the two sets of pixels ('1': use; '0': don't use)
871 # into a final weight matrix to be used in the covariance
872 # calculations below.
873 w = unmasked*wDiff
875 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2):
876 self.log.warning("Number of good points for covariance calculation (%s) is less "
877 "(than threshold %s)", np.sum(w),
878 self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2))
879 return np.nan, np.nan, None, np.nan
881 maxRangeCov = self.config.maximumRangeCovariancesAstier
883 # Calculate covariances via FFT.
884 shapeDiff = np.array(diffIm.image.array.shape)
885 # Calculate the sizes of FFT dimensions.
886 s = shapeDiff + maxRangeCov
887 tempSize = np.array(np.log(s)/np.log(2.)).astype(int)
888 fftSize = np.array(2**(tempSize+1)).astype(int)
889 fftShape = (fftSize[0], fftSize[1])
890 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov)
891 # np.sum(w) is the same as npix[0][0] returned in covDiffAstier
892 try:
893 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov)
894 except ValueError:
895 # This is raised if there are not enough pixels.
896 self.log.warning("Not enough pixels covering the requested covariance range in x/y (%d)",
897 self.config.maximumRangeCovariancesAstier)
898 return np.nan, np.nan, None, np.nan
900 # Compare Cov[0,0] and afwMath.VARIANCECLIP covDiffAstier[0]
901 # is the Cov[0,0] element, [3] is the variance, and there's a
902 # factor of 0.5 difference with afwMath.VARIANCECLIP.
903 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00
904 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5))
905 if fractionalDiff >= thresholdPercentage:
906 self.log.warning("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] "
907 "is more than %f%%: %f", thresholdPercentage, fractionalDiff)
909 return mu, varDiff, covDiffAstier, rowMeanVariance
911 def getImageAreasMasksStats(self, exposure1, exposure2, region=None):
912 """Get image areas in a region as well as masks and statistic objects.
914 Parameters
915 ----------
916 exposure1 : `lsst.afw.image.ExposureF`
917 First exposure of flat field pair.
918 exposure2 : `lsst.afw.image.ExposureF`
919 Second exposure of flat field pair.
920 region : `lsst.geom.Box2I`, optional
921 Region of each exposure where to perform the calculations
922 (e.g, an amplifier).
924 Returns
925 -------
926 im1Area : `lsst.afw.image.MaskedImageF`
927 Masked image from exposure 1.
928 im2Area : `lsst.afw.image.MaskedImageF`
929 Masked image from exposure 2.
930 imStatsCtrl : `lsst.afw.math.StatisticsControl`
931 Statistics control object.
932 mu1 : `float`
933 Clipped mean of im1Area (ADU).
934 mu2 : `float`
935 Clipped mean of im2Area (ADU).
936 """
937 if region is not None:
938 im1Area = exposure1.maskedImage[region]
939 im2Area = exposure2.maskedImage[region]
940 else:
941 im1Area = exposure1.maskedImage
942 im2Area = exposure2.maskedImage
944 # Get mask planes and construct statistics control object from one
945 # of the exposures
946 imMaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
947 imStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
948 self.config.nIterSigmaClipPtc,
949 imMaskVal)
950 imStatsCtrl.setNanSafe(True)
951 imStatsCtrl.setAndMask(imMaskVal)
953 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, imStatsCtrl).getValue()
954 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, imStatsCtrl).getValue()
956 return (im1Area, im2Area, imStatsCtrl, mu1, mu2)
958 def getGainFromFlatPair(self, im1Area, im2Area, imStatsCtrl, mu1, mu2,
959 correctionType='NONE', readNoise=None):
960 """Estimate the gain from a single pair of flats.
962 The basic premise is 1/g = <(I1 - I2)^2/(I1 + I2)> = 1/const,
963 where I1 and I2 correspond to flats 1 and 2, respectively.
964 Corrections for the variable QE and the read-noise are then
965 made following the derivation in Robert Lupton's forthcoming
966 book, which gets
968 1/g = <(I1 - I2)^2/(I1 + I2)> - 1/mu(sigma^2 - 1/2g^2).
970 This is a quadratic equation, whose solutions are given by:
972 g = mu +/- sqrt(2*sigma^2 - 2*const*mu + mu^2)/(2*const*mu*2
973 - 2*sigma^2)
975 where 'mu' is the average signal level and 'sigma' is the
976 amplifier's readnoise. The positive solution will be used.
977 The way the correction is applied depends on the value
978 supplied for correctionType.
980 correctionType is one of ['NONE', 'SIMPLE' or 'FULL']
981 'NONE' : uses the 1/g = <(I1 - I2)^2/(I1 + I2)> formula.
982 'SIMPLE' : uses the gain from the 'NONE' method for the
983 1/2g^2 term.
984 'FULL' : solves the full equation for g, discarding the
985 non-physical solution to the resulting quadratic.
987 Parameters
988 ----------
989 im1Area : `lsst.afw.image.maskedImage.MaskedImageF`
990 Masked image from exposure 1.
991 im2Area : `lsst.afw.image.maskedImage.MaskedImageF`
992 Masked image from exposure 2.
993 imStatsCtrl : `lsst.afw.math.StatisticsControl`
994 Statistics control object.
995 mu1: `float`
996 Clipped mean of im1Area (ADU).
997 mu2: `float`
998 Clipped mean of im2Area (ADU).
999 correctionType : `str`, optional
1000 The correction applied, one of ['NONE', 'SIMPLE', 'FULL']
1001 readNoise : `float`, optional
1002 Amplifier readout noise (ADU).
1004 Returns
1005 -------
1006 gain : `float`
1007 Gain, in e/ADU.
1009 Raises
1010 ------
1011 RuntimeError
1012 Raise if `correctionType` is not one of 'NONE',
1013 'SIMPLE', or 'FULL'.
1014 """
1015 if correctionType not in ['NONE', 'SIMPLE', 'FULL']:
1016 raise RuntimeError("Unknown correction type: %s" % correctionType)
1018 if correctionType != 'NONE' and not np.isfinite(readNoise):
1019 self.log.warning("'correctionType' in 'getGainFromFlatPair' is %s, "
1020 "but 'readNoise' is NaN. Setting 'correctionType' "
1021 "to 'NONE', so a gain value will be estimated without "
1022 "corrections." % correctionType)
1023 correctionType = 'NONE'
1025 mu = 0.5*(mu1 + mu2)
1027 # ratioIm = (I1 - I2)^2 / (I1 + I2)
1028 temp = im2Area.clone()
1029 ratioIm = im1Area.clone()
1030 ratioIm -= temp
1031 ratioIm *= ratioIm
1033 # Sum of pairs
1034 sumIm = im1Area.clone()
1035 sumIm += temp
1037 ratioIm /= sumIm
1039 const = afwMath.makeStatistics(ratioIm, afwMath.MEAN, imStatsCtrl).getValue()
1040 gain = 1. / const
1042 if correctionType == 'SIMPLE':
1043 gain = 1/(const - (1/mu)*(readNoise**2 - (1/2*gain**2)))
1044 elif correctionType == 'FULL':
1045 root = np.sqrt(mu**2 - 2*mu*const + 2*readNoise**2)
1046 denom = (2*const*mu - 2*readNoise**2)
1047 positiveSolution = (root + mu)/denom
1048 gain = positiveSolution
1050 return gain
1052 def getReadNoise(self, exposureMetadata, taskMetadata, ampName):
1053 """Gets readout noise for an amp from ISR metadata.
1055 If possible, this attempts to get the now-standard headers
1056 added to the exposure itself. If not found there, the ISR
1057 TaskMetadata is searched. If neither of these has the value,
1058 warn and set the read noise to NaN.
1060 Parameters
1061 ----------
1062 exposureMetadata : `lsst.daf.base.PropertySet`
1063 Metadata to check for read noise first.
1064 taskMetadata : `lsst.pipe.base.TaskMetadata`
1065 List of exposures metadata from ISR for this exposure.
1066 ampName : `str`
1067 Amplifier name.
1069 Returns
1070 -------
1071 readNoise : `float`
1072 The read noise for this set of exposure/amplifier.
1073 """
1074 # Try from the exposure first.
1075 expectedKey = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {ampName}"
1076 if expectedKey in exposureMetadata:
1077 return exposureMetadata[expectedKey]
1079 # If not, try getting it from the task metadata.
1080 expectedKey = f"RESIDUAL STDEV {ampName}"
1081 if "isr" in taskMetadata:
1082 if expectedKey in taskMetadata["isr"]:
1083 return taskMetadata["isr"][expectedKey]
1085 self.log.warning("Median readout noise from ISR metadata for amp %s "
1086 "could not be calculated." % ampName)
1087 return np.nan
1089 def computeGaussianHistogramParameters(self, im1Area, im2Area, imStatsCtrl, mu1, mu2):
1090 """Compute KS test for a Gaussian model fit to a histogram of the
1091 difference image.
1093 Parameters
1094 ----------
1095 im1Area : `lsst.afw.image.MaskedImageF`
1096 Masked image from exposure 1.
1097 im2Area : `lsst.afw.image.MaskedImageF`
1098 Masked image from exposure 2.
1099 imStatsCtrl : `lsst.afw.math.StatisticsControl`
1100 Statistics control object.
1101 mu1 : `float`
1102 Clipped mean of im1Area (ADU).
1103 mu2 : `float`
1104 Clipped mean of im2Area (ADU).
1106 Returns
1107 -------
1108 varFit : `float`
1109 Variance from the Gaussian fit.
1110 chi2Dof : `float`
1111 Chi-squared per degree of freedom of Gaussian fit.
1112 kspValue : `float`
1113 The KS test p-value for the Gaussian fit.
1115 Notes
1116 -----
1117 The algorithm here was originally developed by Aaron Roodman.
1118 Tests on the full focal plane of LSSTCam during testing has shown
1119 that a KS test p-value cut of 0.01 is a good discriminant for
1120 well-behaved flat pairs (p>0.01) and poorly behaved non-Gaussian
1121 flat pairs (p<0.01).
1122 """
1123 diffExp = im1Area.clone()
1124 diffExp -= im2Area
1126 sel = (((diffExp.mask.array & imStatsCtrl.getAndMask()) == 0)
1127 & np.isfinite(diffExp.mask.array))
1128 diffArr = diffExp.image.array[sel]
1130 numOk = len(diffArr)
1132 if numOk >= self.config.ksHistMinDataValues and np.isfinite(mu1) and np.isfinite(mu2):
1133 # Create a histogram symmetric around zero, with a bin size
1134 # determined from the expected variance given by the average of
1135 # the input signal levels.
1136 lim = self.config.ksHistLimitMultiplier * np.sqrt((mu1 + mu2)/2.)
1137 yVals, binEdges = np.histogram(diffArr, bins=self.config.ksHistNBins, range=[-lim, lim])
1139 # Fit the histogram with a Gaussian model.
1140 model = GaussianModel()
1141 yVals = yVals.astype(np.float64)
1142 xVals = ((binEdges[0: -1] + binEdges[1:])/2.).astype(np.float64)
1143 errVals = np.sqrt(yVals)
1144 errVals[(errVals == 0.0)] = 1.0
1145 pars = model.guess(yVals, x=xVals)
1146 with warnings.catch_warnings():
1147 warnings.simplefilter("ignore")
1148 # The least-squares fitter sometimes spouts (spurious) warnings
1149 # when the model is very bad. Swallow these warnings now and
1150 # let the KS test check the model below.
1151 out = model.fit(
1152 yVals,
1153 pars,
1154 x=xVals,
1155 weights=1./errVals,
1156 calc_covar=True,
1157 method="least_squares",
1158 )
1160 # Calculate chi2.
1161 chiArr = out.residual
1162 nDof = len(yVals) - 3
1163 chi2Dof = np.sum(chiArr**2.)/nDof
1164 sigmaFit = out.params["sigma"].value
1166 # Calculate KS test p-value for the fit.
1167 ksResult = scipy.stats.ks_1samp(
1168 diffArr,
1169 scipy.stats.norm.cdf,
1170 (out.params["center"].value, sigmaFit),
1171 )
1173 kspValue = ksResult.pvalue
1174 if kspValue < 1e-15:
1175 kspValue = 0.0
1177 varFit = sigmaFit**2.
1179 else:
1180 varFit = np.nan
1181 chi2Dof = np.nan
1182 kspValue = 0.0
1184 return varFit, chi2Dof, kspValue