Coverage for python/lsst/ip/isr/isrStatistics.py: 19%
153 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-09 12:50 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-09 12:50 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
24import numpy as np
26from scipy.signal.windows import hamming, hann, gaussian
28import lsst.afw.math as afwMath
29import lsst.afw.image as afwImage
30import lsst.pipe.base as pipeBase
31import lsst.pex.config as pexConfig
33from lsst.afw.cameraGeom import ReadoutCorner
36class IsrStatisticsTaskConfig(pexConfig.Config):
37 """Image statistics options.
38 """
39 doCtiStatistics = pexConfig.Field(
40 dtype=bool,
41 doc="Measure CTI statistics from image and overscans?",
42 default=False,
43 )
44 doApplyGainsForCtiStatistics = pexConfig.Field(
45 dtype=bool,
46 doc="Apply gain to the overscan region when measuring CTI statistics?",
47 default=True,
48 )
50 doBandingStatistics = pexConfig.Field(
51 dtype=bool,
52 doc="Measure image banding metric?",
53 default=False,
54 )
55 bandingKernelSize = pexConfig.Field( 55 ↛ exitline 55 didn't jump to the function exit
56 dtype=int,
57 doc="Width of box for boxcar smoothing for banding metric.",
58 default=3,
59 check=lambda x: x == 0 or x % 2 != 0,
60 )
61 bandingFractionLow = pexConfig.Field( 61 ↛ exitline 61 didn't jump to the function exit
62 dtype=float,
63 doc="Fraction of values to exclude from low samples.",
64 default=0.1,
65 check=lambda x: x >= 0.0 and x <= 1.0
66 )
67 bandingFractionHigh = pexConfig.Field( 67 ↛ exitline 67 didn't jump to the function exit
68 dtype=float,
69 doc="Fraction of values to exclude from high samples.",
70 default=0.9,
71 check=lambda x: x >= 0.0 and x <= 1.0,
72 )
73 bandingUseHalfDetector = pexConfig.Field(
74 dtype=float,
75 doc="Use only the first half set of amplifiers.",
76 default=True,
77 )
79 doProjectionStatistics = pexConfig.Field(
80 dtype=bool,
81 doc="Measure projection metric?",
82 default=False,
83 )
84 projectionKernelSize = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit
85 dtype=int,
86 doc="Width of box for boxcar smoothing of projections.",
87 default=0,
88 check=lambda x: x == 0 or x % 2 != 0,
89 )
90 doProjectionFft = pexConfig.Field(
91 dtype=bool,
92 doc="Generate FFTs from the image projections?",
93 default=False,
94 )
95 projectionFftWindow = pexConfig.ChoiceField(
96 dtype=str,
97 doc="Type of windowing to use prior to calculating FFT.",
98 default="HAMMING",
99 allowed={
100 "HAMMING": "Hamming window.",
101 "HANN": "Hann window.",
102 "GAUSSIAN": "Gaussian window.",
103 "NONE": "No window."
104 }
105 )
107 stat = pexConfig.Field(
108 dtype=str,
109 default='MEANCLIP',
110 doc="Statistic name to use to measure regions.",
111 )
112 nSigmaClip = pexConfig.Field(
113 dtype=float,
114 default=3.0,
115 doc="Clipping threshold for background",
116 )
117 nIter = pexConfig.Field(
118 dtype=int,
119 default=3,
120 doc="Clipping iterations for background",
121 )
122 badMask = pexConfig.ListField(
123 dtype=str,
124 default=["BAD", "INTRP", "SAT"],
125 doc="Mask planes to ignore when identifying source pixels."
126 )
129class IsrStatisticsTask(pipeBase.Task):
130 """Task to measure arbitrary statistics on ISR processed exposures.
132 The goal is to wrap a number of optional measurements that are
133 useful for calibration production and detector stability.
134 """
135 ConfigClass = IsrStatisticsTaskConfig
136 _DefaultName = "isrStatistics"
138 def __init__(self, statControl=None, **kwargs):
139 super().__init__(**kwargs)
140 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
141 afwImage.Mask.getPlaneBitMask(self.config.badMask))
142 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
144 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
145 """Task to run arbitrary statistics.
147 The statistics should be measured by individual methods, and
148 add to the dictionary in the return struct.
150 Parameters
151 ----------
152 inputExp : `lsst.afw.image.Exposure`
153 The exposure to measure.
154 ptc : `lsst.ip.isr.PtcDataset`, optional
155 A PTC object containing gains to use.
156 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
157 List of overscan results. Expected fields are:
159 ``imageFit``
160 Value or fit subtracted from the amplifier image data
161 (scalar or `lsst.afw.image.Image`).
162 ``overscanFit``
163 Value or fit subtracted from the overscan image data
164 (scalar or `lsst.afw.image.Image`).
165 ``overscanImage``
166 Image of the overscan region with the overscan
167 correction applied (`lsst.afw.image.Image`). This
168 quantity is used to estimate the amplifier read noise
169 empirically.
171 Returns
172 -------
173 resultStruct : `lsst.pipe.base.Struct`
174 Contains the measured statistics as a dict stored in a
175 field named ``results``.
177 Raises
178 ------
179 RuntimeError
180 Raised if the amplifier gains could not be found.
181 """
182 # Find gains.
183 detector = inputExp.getDetector()
184 if ptc is not None:
185 gains = ptc.gain
186 elif detector is not None:
187 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
188 else:
189 raise RuntimeError("No source of gains provided.")
191 ctiResults = None
192 if self.config.doCtiStatistics:
193 ctiResults = self.measureCti(inputExp, overscanResults, gains)
195 bandingResults = None
196 if self.config.doBandingStatistics:
197 bandingResults = self.measureBanding(inputExp, overscanResults)
199 projectionResults = None
200 if self.config.doProjectionStatistics:
201 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
203 return pipeBase.Struct(
204 results={'CTI': ctiResults,
205 'BANDING': bandingResults,
206 'PROJECTION': projectionResults,
207 },
208 )
210 def measureCti(self, inputExp, overscans, gains):
211 """Task to measure CTI statistics.
213 Parameters
214 ----------
215 inputExp : `lsst.afw.image.Exposure`
216 Exposure to measure.
217 overscans : `list` [`lsst.pipe.base.Struct`]
218 List of overscan results. Expected fields are:
220 ``imageFit``
221 Value or fit subtracted from the amplifier image data
222 (scalar or `lsst.afw.image.Image`).
223 ``overscanFit``
224 Value or fit subtracted from the overscan image data
225 (scalar or `lsst.afw.image.Image`).
226 ``overscanImage``
227 Image of the overscan region with the overscan
228 correction applied (`lsst.afw.image.Image`). This
229 quantity is used to estimate the amplifier read noise
230 empirically.
231 gains : `dict` [`str` `float`]
232 Dictionary of per-amplifier gains, indexed by amplifier name.
234 Returns
235 -------
236 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
237 Dictionary of measurements, keyed by amplifier name and
238 statistics segment.
239 """
240 outputStats = {}
242 detector = inputExp.getDetector()
243 image = inputExp.image
245 # Ensure we have the same number of overscans as amplifiers.
246 assert len(overscans) == len(detector.getAmplifiers())
248 for ampIter, amp in enumerate(detector.getAmplifiers()):
249 ampStats = {}
250 gain = gains[amp.getName()]
251 readoutCorner = amp.getReadoutCorner()
252 # Full data region.
253 dataRegion = image[amp.getBBox()]
254 ampStats['IMAGE_MEAN'] = afwMath.makeStatistics(dataRegion, self.statType,
255 self.statControl).getValue()
257 # First and last image columns.
258 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
259 self.statType,
260 self.statControl).getValue()
261 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
262 self.statType,
263 self.statControl).getValue()
265 # We want these relative to the readout corner. If that's
266 # on the right side, we need to swap them.
267 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
268 ampStats['FIRST_MEAN'] = pixelZ
269 ampStats['LAST_MEAN'] = pixelA
270 else:
271 ampStats['FIRST_MEAN'] = pixelA
272 ampStats['LAST_MEAN'] = pixelZ
274 # Measure the columns of the overscan.
275 if overscans[ampIter] is None:
276 # The amplifier is likely entirely bad, and needs to
277 # be skipped.
278 self.log.warn("No overscan information available for ISR statistics for amp %s.",
279 amp.getName())
280 nCols = amp.getSerialOverscanBBox().getWidth()
281 ampStats['OVERSCAN_COLUMNS'] = np.full((nCols, ), np.nan)
282 ampStats['OVERSCAN_VALUES'] = np.full((nCols, ), np.nan)
283 else:
284 overscanImage = overscans[ampIter].overscanImage
285 columns = []
286 values = []
287 for column in range(0, overscanImage.getWidth()):
288 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column],
289 self.statType, self.statControl).getValue()
290 columns.append(column)
291 if self.config.doApplyGainsForCtiStatistics:
292 values.append(gain * osMean)
293 else:
294 values.append(osMean)
296 # We want these relative to the readout corner. If that's
297 # on the right side, we need to swap them.
298 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
299 ampStats['OVERSCAN_COLUMNS'] = list(reversed(columns))
300 ampStats['OVERSCAN_VALUES'] = list(reversed(values))
301 else:
302 ampStats['OVERSCAN_COLUMNS'] = columns
303 ampStats['OVERSCAN_VALUES'] = values
305 outputStats[amp.getName()] = ampStats
307 return outputStats
309 @staticmethod
310 def makeKernel(kernelSize):
311 """Make a boxcar smoothing kernel.
313 Parameters
314 ----------
315 kernelSize : `int`
316 Size of the kernel in pixels.
318 Returns
319 -------
320 kernel : `np.array`
321 Kernel for boxcar smoothing.
322 """
323 if kernelSize > 0:
324 kernel = np.full(kernelSize, 1.0 / kernelSize)
325 else:
326 kernel = np.array([1.0])
327 return kernel
329 def measureBanding(self, inputExp, overscans):
330 """Task to measure banding statistics.
332 Parameters
333 ----------
334 inputExp : `lsst.afw.image.Exposure`
335 Exposure to measure.
336 overscans : `list` [`lsst.pipe.base.Struct`]
337 List of overscan results. Expected fields are:
339 ``imageFit``
340 Value or fit subtracted from the amplifier image data
341 (scalar or `lsst.afw.image.Image`).
342 ``overscanFit``
343 Value or fit subtracted from the overscan image data
344 (scalar or `lsst.afw.image.Image`).
345 ``overscanImage``
346 Image of the overscan region with the overscan
347 correction applied (`lsst.afw.image.Image`). This
348 quantity is used to estimate the amplifier read noise
349 empirically.
351 Returns
352 -------
353 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
354 Dictionary of measurements, keyed by amplifier name and
355 statistics segment.
356 """
357 outputStats = {}
359 detector = inputExp.getDetector()
360 kernel = self.makeKernel(self.config.bandingKernelSize)
362 outputStats['AMP_BANDING'] = []
363 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
364 overscanFit = np.array(overscanData.overscanFit)
365 overscanArray = overscanData.overscanImage.image.array
366 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
368 smoothedOverscan = np.convolve(rawOverscan, kernel, mode='valid')
370 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
371 self.config.bandingFractionHigh])
372 outputStats['AMP_BANDING'].append(float(high - low))
374 if self.config.bandingUseHalfDetector:
375 fullLength = len(outputStats['AMP_BANDING'])
376 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING'][0:fullLength//2]))
377 else:
378 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING']))
380 return outputStats
382 def measureProjectionStatistics(self, inputExp, overscans):
383 """Task to measure metrics from image slicing.
385 Parameters
386 ----------
387 inputExp : `lsst.afw.image.Exposure`
388 Exposure to measure.
389 overscans : `list` [`lsst.pipe.base.Struct`]
390 List of overscan results. Expected fields are:
392 ``imageFit``
393 Value or fit subtracted from the amplifier image data
394 (scalar or `lsst.afw.image.Image`).
395 ``overscanFit``
396 Value or fit subtracted from the overscan image data
397 (scalar or `lsst.afw.image.Image`).
398 ``overscanImage``
399 Image of the overscan region with the overscan
400 correction applied (`lsst.afw.image.Image`). This
401 quantity is used to estimate the amplifier read noise
402 empirically.
404 Returns
405 -------
406 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
407 Dictionary of measurements, keyed by amplifier name and
408 statistics segment.
409 """
410 outputStats = {}
412 detector = inputExp.getDetector()
413 kernel = self.makeKernel(self.config.projectionKernelSize)
415 outputStats['AMP_VPROJECTION'] = {}
416 outputStats['AMP_HPROJECTION'] = {}
417 convolveMode = 'valid'
418 if self.config.doProjectionFft:
419 outputStats['AMP_VFFT_REAL'] = {}
420 outputStats['AMP_VFFT_IMAG'] = {}
421 outputStats['AMP_HFFT_REAL'] = {}
422 outputStats['AMP_HFFT_IMAG'] = {}
423 convolveMode = 'same'
425 for amp in detector.getAmplifiers():
426 ampArray = inputExp.image[amp.getBBox()].array
428 horizontalProjection = np.mean(ampArray, axis=0)
429 verticalProjection = np.mean(ampArray, axis=1)
431 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
432 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
434 outputStats['AMP_HPROJECTION'][amp.getName()] = horizontalProjection.tolist()
435 outputStats['AMP_VPROJECTION'][amp.getName()] = verticalProjection.tolist()
437 if self.config.doProjectionFft:
438 horizontalWindow = np.ones_like(horizontalProjection)
439 verticalWindow = np.ones_like(verticalProjection)
440 if self.config.projectionFftWindow == "NONE":
441 pass
442 elif self.config.projectionFftWindow == "HAMMING":
443 horizontalWindow = hamming(len(horizontalProjection))
444 verticalWindow = hamming(len(verticalProjection))
445 elif self.config.projectionFftWindow == "HANN":
446 horizontalWindow = hann(len(horizontalProjection))
447 verticalWindow = hann(len(verticalProjection))
448 elif self.config.projectionFftWindow == "GAUSSIAN":
449 horizontalWindow = gaussian(len(horizontalProjection))
450 verticalWindow = gaussian(len(verticalProjection))
451 else:
452 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
454 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
455 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
456 outputStats['AMP_HFFT_REAL'][amp.getName()] = np.real(horizontalFFT).tolist()
457 outputStats['AMP_HFFT_IMAG'][amp.getName()] = np.imag(horizontalFFT).tolist()
458 outputStats['AMP_VFFT_REAL'][amp.getName()] = np.real(verticalFFT).tolist()
459 outputStats['AMP_VFFT_IMAG'][amp.getName()] = np.imag(verticalFFT).tolist()
461 return outputStats