Coverage for python/lsst/ip/isr/isrStatistics.py: 18%
170 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-23 11:36 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-23 11:36 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
24import numpy as np
26from scipy.signal.windows import hamming, hann, gaussian
28import lsst.afw.math as afwMath
29import lsst.afw.image as afwImage
30import lsst.pipe.base as pipeBase
31import lsst.pex.config as pexConfig
33from lsst.afw.cameraGeom import ReadoutCorner
36class IsrStatisticsTaskConfig(pexConfig.Config):
37 """Image statistics options.
38 """
39 doCtiStatistics = pexConfig.Field(
40 dtype=bool,
41 doc="Measure CTI statistics from image and overscans?",
42 default=False,
43 )
44 doApplyGainsForCtiStatistics = pexConfig.Field(
45 dtype=bool,
46 doc="Apply gain to the overscan region when measuring CTI statistics?",
47 default=True,
48 )
50 doBandingStatistics = pexConfig.Field(
51 dtype=bool,
52 doc="Measure image banding metric?",
53 default=False,
54 )
55 bandingKernelSize = pexConfig.Field( 55 ↛ exitline 55 didn't jump to the function exit
56 dtype=int,
57 doc="Width of box for boxcar smoothing for banding metric.",
58 default=3,
59 check=lambda x: x == 0 or x % 2 != 0,
60 )
61 bandingFractionLow = pexConfig.Field( 61 ↛ exitline 61 didn't jump to the function exit
62 dtype=float,
63 doc="Fraction of values to exclude from low samples.",
64 default=0.1,
65 check=lambda x: x >= 0.0 and x <= 1.0
66 )
67 bandingFractionHigh = pexConfig.Field( 67 ↛ exitline 67 didn't jump to the function exit
68 dtype=float,
69 doc="Fraction of values to exclude from high samples.",
70 default=0.9,
71 check=lambda x: x >= 0.0 and x <= 1.0,
72 )
73 bandingUseHalfDetector = pexConfig.Field(
74 dtype=float,
75 doc="Use only the first half set of amplifiers.",
76 default=True,
77 )
79 doProjectionStatistics = pexConfig.Field(
80 dtype=bool,
81 doc="Measure projection metric?",
82 default=False,
83 )
84 projectionKernelSize = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit
85 dtype=int,
86 doc="Width of box for boxcar smoothing of projections.",
87 default=0,
88 check=lambda x: x == 0 or x % 2 != 0,
89 )
90 doProjectionFft = pexConfig.Field(
91 dtype=bool,
92 doc="Generate FFTs from the image projections?",
93 default=False,
94 )
95 projectionFftWindow = pexConfig.ChoiceField(
96 dtype=str,
97 doc="Type of windowing to use prior to calculating FFT.",
98 default="HAMMING",
99 allowed={
100 "HAMMING": "Hamming window.",
101 "HANN": "Hann window.",
102 "GAUSSIAN": "Gaussian window.",
103 "NONE": "No window."
104 }
105 )
107 doCopyCalibDistributionStatistics = pexConfig.Field(
108 dtype=bool,
109 doc="Copy calibration distribution statistics to output?",
110 default=False,
111 )
112 expectedDistributionLevels = pexConfig.ListField(
113 dtype=float,
114 doc="Percentile levels expected in the calibration header.",
115 default=[0, 5, 16, 50, 84, 95, 100],
116 )
118 stat = pexConfig.Field(
119 dtype=str,
120 default="MEANCLIP",
121 doc="Statistic name to use to measure regions.",
122 )
123 nSigmaClip = pexConfig.Field(
124 dtype=float,
125 default=3.0,
126 doc="Clipping threshold for background",
127 )
128 nIter = pexConfig.Field(
129 dtype=int,
130 default=3,
131 doc="Clipping iterations for background",
132 )
133 badMask = pexConfig.ListField(
134 dtype=str,
135 default=["BAD", "INTRP", "SAT"],
136 doc="Mask planes to ignore when identifying source pixels."
137 )
140class IsrStatisticsTask(pipeBase.Task):
141 """Task to measure arbitrary statistics on ISR processed exposures.
143 The goal is to wrap a number of optional measurements that are
144 useful for calibration production and detector stability.
145 """
146 ConfigClass = IsrStatisticsTaskConfig
147 _DefaultName = "isrStatistics"
149 def __init__(self, statControl=None, **kwargs):
150 super().__init__(**kwargs)
151 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
152 afwImage.Mask.getPlaneBitMask(self.config.badMask))
153 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
155 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
156 """Task to run arbitrary statistics.
158 The statistics should be measured by individual methods, and
159 add to the dictionary in the return struct.
161 Parameters
162 ----------
163 inputExp : `lsst.afw.image.Exposure`
164 The exposure to measure.
165 ptc : `lsst.ip.isr.PtcDataset`, optional
166 A PTC object containing gains to use.
167 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
168 List of overscan results. Expected fields are:
170 ``imageFit``
171 Value or fit subtracted from the amplifier image data
172 (scalar or `lsst.afw.image.Image`).
173 ``overscanFit``
174 Value or fit subtracted from the overscan image data
175 (scalar or `lsst.afw.image.Image`).
176 ``overscanImage``
177 Image of the overscan region with the overscan
178 correction applied (`lsst.afw.image.Image`). This
179 quantity is used to estimate the amplifier read noise
180 empirically.
182 Returns
183 -------
184 resultStruct : `lsst.pipe.base.Struct`
185 Contains the measured statistics as a dict stored in a
186 field named ``results``.
188 Raises
189 ------
190 RuntimeError
191 Raised if the amplifier gains could not be found.
192 """
193 # Find gains.
194 detector = inputExp.getDetector()
195 if ptc is not None:
196 gains = ptc.gain
197 elif detector is not None:
198 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
199 else:
200 raise RuntimeError("No source of gains provided.")
202 ctiResults = None
203 if self.config.doCtiStatistics:
204 ctiResults = self.measureCti(inputExp, overscanResults, gains)
206 bandingResults = None
207 if self.config.doBandingStatistics:
208 bandingResults = self.measureBanding(inputExp, overscanResults)
210 projectionResults = None
211 if self.config.doProjectionStatistics:
212 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
214 calibDistributionResults = None
215 if self.config.doCopyCalibDistributionStatistics:
216 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs)
218 return pipeBase.Struct(
219 results={"CTI": ctiResults,
220 "BANDING": bandingResults,
221 "PROJECTION": projectionResults,
222 "CALIBDIST": calibDistributionResults,
223 },
224 )
226 def measureCti(self, inputExp, overscans, gains):
227 """Task to measure CTI statistics.
229 Parameters
230 ----------
231 inputExp : `lsst.afw.image.Exposure`
232 Exposure to measure.
233 overscans : `list` [`lsst.pipe.base.Struct`]
234 List of overscan results. Expected fields are:
236 ``imageFit``
237 Value or fit subtracted from the amplifier image data
238 (scalar or `lsst.afw.image.Image`).
239 ``overscanFit``
240 Value or fit subtracted from the overscan image data
241 (scalar or `lsst.afw.image.Image`).
242 ``overscanImage``
243 Image of the overscan region with the overscan
244 correction applied (`lsst.afw.image.Image`). This
245 quantity is used to estimate the amplifier read noise
246 empirically.
247 gains : `dict` [`str` `float`]
248 Dictionary of per-amplifier gains, indexed by amplifier name.
250 Returns
251 -------
252 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
253 Dictionary of measurements, keyed by amplifier name and
254 statistics segment.
255 """
256 outputStats = {}
258 detector = inputExp.getDetector()
259 image = inputExp.image
261 # Ensure we have the same number of overscans as amplifiers.
262 assert len(overscans) == len(detector.getAmplifiers())
264 for ampIter, amp in enumerate(detector.getAmplifiers()):
265 ampStats = {}
266 gain = gains[amp.getName()]
267 readoutCorner = amp.getReadoutCorner()
268 # Full data region.
269 dataRegion = image[amp.getBBox()]
270 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType,
271 self.statControl).getValue()
273 # First and last image columns.
274 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
275 self.statType,
276 self.statControl).getValue()
277 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
278 self.statType,
279 self.statControl).getValue()
281 # We want these relative to the readout corner. If that's
282 # on the right side, we need to swap them.
283 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
284 ampStats["FIRST_MEAN"] = pixelZ
285 ampStats["LAST_MEAN"] = pixelA
286 else:
287 ampStats["FIRST_MEAN"] = pixelA
288 ampStats["LAST_MEAN"] = pixelZ
290 # Measure the columns of the overscan.
291 if overscans[ampIter] is None:
292 # The amplifier is likely entirely bad, and needs to
293 # be skipped.
294 self.log.warning("No overscan information available for ISR statistics for amp %s.",
295 amp.getName())
296 nCols = amp.getSerialOverscanBBox().getWidth()
297 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
298 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
299 else:
300 overscanImage = overscans[ampIter].overscanImage
301 columns = []
302 values = []
303 for column in range(0, overscanImage.getWidth()):
304 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column],
305 self.statType, self.statControl).getValue()
306 columns.append(column)
307 if self.config.doApplyGainsForCtiStatistics:
308 values.append(gain * osMean)
309 else:
310 values.append(osMean)
312 # We want these relative to the readout corner. If that's
313 # on the right side, we need to swap them.
314 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
315 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns))
316 ampStats["OVERSCAN_VALUES"] = list(reversed(values))
317 else:
318 ampStats["OVERSCAN_COLUMNS"] = columns
319 ampStats["OVERSCAN_VALUES"] = values
321 outputStats[amp.getName()] = ampStats
323 return outputStats
325 @staticmethod
326 def makeKernel(kernelSize):
327 """Make a boxcar smoothing kernel.
329 Parameters
330 ----------
331 kernelSize : `int`
332 Size of the kernel in pixels.
334 Returns
335 -------
336 kernel : `np.array`
337 Kernel for boxcar smoothing.
338 """
339 if kernelSize > 0:
340 kernel = np.full(kernelSize, 1.0 / kernelSize)
341 else:
342 kernel = np.array([1.0])
343 return kernel
345 def measureBanding(self, inputExp, overscans):
346 """Task to measure banding statistics.
348 Parameters
349 ----------
350 inputExp : `lsst.afw.image.Exposure`
351 Exposure to measure.
352 overscans : `list` [`lsst.pipe.base.Struct`]
353 List of overscan results. Expected fields are:
355 ``imageFit``
356 Value or fit subtracted from the amplifier image data
357 (scalar or `lsst.afw.image.Image`).
358 ``overscanFit``
359 Value or fit subtracted from the overscan image data
360 (scalar or `lsst.afw.image.Image`).
361 ``overscanImage``
362 Image of the overscan region with the overscan
363 correction applied (`lsst.afw.image.Image`). This
364 quantity is used to estimate the amplifier read noise
365 empirically.
367 Returns
368 -------
369 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
370 Dictionary of measurements, keyed by amplifier name and
371 statistics segment.
372 """
373 outputStats = {}
375 detector = inputExp.getDetector()
376 kernel = self.makeKernel(self.config.bandingKernelSize)
378 outputStats["AMP_BANDING"] = []
379 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
380 overscanFit = np.array(overscanData.overscanFit)
381 overscanArray = overscanData.overscanImage.image.array
382 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
384 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid")
386 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
387 self.config.bandingFractionHigh])
388 outputStats["AMP_BANDING"].append(float(high - low))
390 if self.config.bandingUseHalfDetector:
391 fullLength = len(outputStats["AMP_BANDING"])
392 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2]))
393 else:
394 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"]))
396 return outputStats
398 def measureProjectionStatistics(self, inputExp, overscans):
399 """Task to measure metrics from image slicing.
401 Parameters
402 ----------
403 inputExp : `lsst.afw.image.Exposure`
404 Exposure to measure.
405 overscans : `list` [`lsst.pipe.base.Struct`]
406 List of overscan results. Expected fields are:
408 ``imageFit``
409 Value or fit subtracted from the amplifier image data
410 (scalar or `lsst.afw.image.Image`).
411 ``overscanFit``
412 Value or fit subtracted from the overscan image data
413 (scalar or `lsst.afw.image.Image`).
414 ``overscanImage``
415 Image of the overscan region with the overscan
416 correction applied (`lsst.afw.image.Image`). This
417 quantity is used to estimate the amplifier read noise
418 empirically.
420 Returns
421 -------
422 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
423 Dictionary of measurements, keyed by amplifier name and
424 statistics segment.
425 """
426 outputStats = {}
428 detector = inputExp.getDetector()
429 kernel = self.makeKernel(self.config.projectionKernelSize)
431 outputStats["AMP_VPROJECTION"] = {}
432 outputStats["AMP_HPROJECTION"] = {}
433 convolveMode = "valid"
434 if self.config.doProjectionFft:
435 outputStats["AMP_VFFT_REAL"] = {}
436 outputStats["AMP_VFFT_IMAG"] = {}
437 outputStats["AMP_HFFT_REAL"] = {}
438 outputStats["AMP_HFFT_IMAG"] = {}
439 convolveMode = "same"
441 for amp in detector.getAmplifiers():
442 ampArray = inputExp.image[amp.getBBox()].array
444 horizontalProjection = np.mean(ampArray, axis=0)
445 verticalProjection = np.mean(ampArray, axis=1)
447 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
448 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
450 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
451 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
453 if self.config.doProjectionFft:
454 horizontalWindow = np.ones_like(horizontalProjection)
455 verticalWindow = np.ones_like(verticalProjection)
456 if self.config.projectionFftWindow == "NONE":
457 pass
458 elif self.config.projectionFftWindow == "HAMMING":
459 horizontalWindow = hamming(len(horizontalProjection))
460 verticalWindow = hamming(len(verticalProjection))
461 elif self.config.projectionFftWindow == "HANN":
462 horizontalWindow = hann(len(horizontalProjection))
463 verticalWindow = hann(len(verticalProjection))
464 elif self.config.projectionFftWindow == "GAUSSIAN":
465 horizontalWindow = gaussian(len(horizontalProjection))
466 verticalWindow = gaussian(len(verticalProjection))
467 else:
468 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
470 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
471 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
472 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
473 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
474 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
475 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
477 return outputStats
479 def copyCalibDistributionStatistics(self, inputExp, **kwargs):
480 """Copy calibration statistics for this exposure.
482 Parameters
483 ----------
484 inputExp : `lsst.afw.image.Exposure`
485 The exposure being processed.
486 **kwargs :
487 Keyword arguments with calibrations.
489 Returns
490 -------
491 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
492 Dictionary of measurements, keyed by amplifier name and
493 statistics segment.
494 """
495 outputStats = {}
497 for amp in inputExp.getDetector():
498 ampStats = {}
500 for calibType in ("bias", "dark", "flat"):
501 if kwargs.get(calibType, None) is not None:
502 metadata = kwargs[calibType].getMetadata()
503 for pct in self.config.expectedDistributionLevels:
504 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
505 ampStats[key] = metadata.get(key, np.nan)
506 outputStats[amp.getName()] = ampStats
508 return outputStats