Coverage for python/lsst/ip/isr/isrStatistics.py: 18%
150 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-28 11:40 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-28 11:40 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
24import numpy as np
26from scipy.signal.windows import hamming, hann, gaussian
28import lsst.afw.math as afwMath
29import lsst.afw.image as afwImage
30import lsst.pipe.base as pipeBase
31import lsst.pex.config as pexConfig
33from lsst.afw.cameraGeom import ReadoutCorner
36class IsrStatisticsTaskConfig(pexConfig.Config):
37 """Image statistics options.
38 """
39 doCtiStatistics = pexConfig.Field(
40 dtype=bool,
41 doc="Measure CTI statistics from image and overscans?",
42 default=False,
43 )
45 doBandingStatistics = pexConfig.Field(
46 dtype=bool,
47 doc="Measure image banding metric?",
48 default=False,
49 )
50 bandingKernelSize = pexConfig.Field( 50 ↛ exitline 50 didn't jump to the function exit
51 dtype=int,
52 doc="Width of box for boxcar smoothing for banding metric.",
53 default=3,
54 check=lambda x: x == 0 or x % 2 != 0,
55 )
56 bandingFractionLow = pexConfig.Field( 56 ↛ exitline 56 didn't jump to the function exit
57 dtype=float,
58 doc="Fraction of values to exclude from low samples.",
59 default=0.1,
60 check=lambda x: x >= 0.0 and x <= 1.0
61 )
62 bandingFractionHigh = pexConfig.Field( 62 ↛ exitline 62 didn't jump to the function exit
63 dtype=float,
64 doc="Fraction of values to exclude from high samples.",
65 default=0.9,
66 check=lambda x: x >= 0.0 and x <= 1.0,
67 )
68 bandingUseHalfDetector = pexConfig.Field(
69 dtype=float,
70 doc="Use only the first half set of amplifiers.",
71 default=True,
72 )
74 doProjectionStatistics = pexConfig.Field(
75 dtype=bool,
76 doc="Measure projection metric?",
77 default=False,
78 )
79 projectionKernelSize = pexConfig.Field( 79 ↛ exitline 79 didn't jump to the function exit
80 dtype=int,
81 doc="Width of box for boxcar smoothing of projections.",
82 default=0,
83 check=lambda x: x == 0 or x % 2 != 0,
84 )
85 doProjectionFft = pexConfig.Field(
86 dtype=bool,
87 doc="Generate FFTs from the image projections?",
88 default=False,
89 )
90 projectionFftWindow = pexConfig.ChoiceField(
91 dtype=str,
92 doc="Type of windowing to use prior to calculating FFT.",
93 default="HAMMING",
94 allowed={
95 "HAMMING": "Hamming window.",
96 "HANN": "Hann window.",
97 "GAUSSIAN": "Gaussian window.",
98 "NONE": "No window."
99 }
100 )
102 stat = pexConfig.Field(
103 dtype=str,
104 default='MEANCLIP',
105 doc="Statistic name to use to measure regions.",
106 )
107 nSigmaClip = pexConfig.Field(
108 dtype=float,
109 default=3.0,
110 doc="Clipping threshold for background",
111 )
112 nIter = pexConfig.Field(
113 dtype=int,
114 default=3,
115 doc="Clipping iterations for background",
116 )
117 badMask = pexConfig.ListField(
118 dtype=str,
119 default=["BAD", "INTRP", "SAT"],
120 doc="Mask planes to ignore when identifying source pixels."
121 )
124class IsrStatisticsTask(pipeBase.Task):
125 """Task to measure arbitrary statistics on ISR processed exposures.
127 The goal is to wrap a number of optional measurements that are
128 useful for calibration production and detector stability.
129 """
130 ConfigClass = IsrStatisticsTaskConfig
131 _DefaultName = "isrStatistics"
133 def __init__(self, statControl=None, **kwargs):
134 super().__init__(**kwargs)
135 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
136 afwImage.Mask.getPlaneBitMask(self.config.badMask))
137 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
139 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
140 """Task to run arbitrary statistics.
142 The statistics should be measured by individual methods, and
143 add to the dictionary in the return struct.
145 Parameters
146 ----------
147 inputExp : `lsst.afw.image.Exposure`
148 The exposure to measure.
149 ptc : `lsst.ip.isr.PtcDataset`, optional
150 A PTC object containing gains to use.
151 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
152 List of overscan results. Expected fields are:
154 ``imageFit``
155 Value or fit subtracted from the amplifier image data
156 (scalar or `lsst.afw.image.Image`).
157 ``overscanFit``
158 Value or fit subtracted from the overscan image data
159 (scalar or `lsst.afw.image.Image`).
160 ``overscanImage``
161 Image of the overscan region with the overscan
162 correction applied (`lsst.afw.image.Image`). This
163 quantity is used to estimate the amplifier read noise
164 empirically.
166 Returns
167 -------
168 resultStruct : `lsst.pipe.base.Struct`
169 Contains the measured statistics as a dict stored in a
170 field named ``results``.
172 Raises
173 ------
174 RuntimeError
175 Raised if the amplifier gains could not be found.
176 """
177 # Find gains.
178 detector = inputExp.getDetector()
179 if ptc is not None:
180 gains = ptc.gain
181 elif detector is not None:
182 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
183 else:
184 raise RuntimeError("No source of gains provided.")
186 ctiResults = None
187 if self.config.doCtiStatistics:
188 ctiResults = self.measureCti(inputExp, overscanResults, gains)
190 bandingResults = None
191 if self.config.doBandingStatistics:
192 bandingResults = self.measureBanding(inputExp, overscanResults)
194 projectionResults = None
195 if self.config.doProjectionStatistics:
196 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
198 return pipeBase.Struct(
199 results={'CTI': ctiResults,
200 'BANDING': bandingResults,
201 'PROJECTION': projectionResults,
202 },
203 )
205 def measureCti(self, inputExp, overscans, gains):
206 """Task to measure CTI statistics.
208 Parameters
209 ----------
210 inputExp : `lsst.afw.image.Exposure`
211 Exposure to measure.
212 overscans : `list` [`lsst.pipe.base.Struct`]
213 List of overscan results. Expected fields are:
215 ``imageFit``
216 Value or fit subtracted from the amplifier image data
217 (scalar or `lsst.afw.image.Image`).
218 ``overscanFit``
219 Value or fit subtracted from the overscan image data
220 (scalar or `lsst.afw.image.Image`).
221 ``overscanImage``
222 Image of the overscan region with the overscan
223 correction applied (`lsst.afw.image.Image`). This
224 quantity is used to estimate the amplifier read noise
225 empirically.
226 gains : `dict` [`str` `float`]
227 Dictionary of per-amplifier gains, indexed by amplifier name.
229 Returns
230 -------
231 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
232 Dictionary of measurements, keyed by amplifier name and
233 statistics segment.
234 """
235 outputStats = {}
237 detector = inputExp.getDetector()
238 image = inputExp.image
240 # Ensure we have the same number of overscans as amplifiers.
241 assert len(overscans) == len(detector.getAmplifiers())
243 for ampIter, amp in enumerate(detector.getAmplifiers()):
244 ampStats = {}
245 gain = gains[amp.getName()]
246 readoutCorner = amp.getReadoutCorner()
247 # Full data region.
248 dataRegion = image[amp.getBBox()]
249 ampStats['IMAGE_MEAN'] = afwMath.makeStatistics(dataRegion, self.statType,
250 self.statControl).getValue()
252 # First and last image columns.
253 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
254 self.statType,
255 self.statControl).getValue()
256 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
257 self.statType,
258 self.statControl).getValue()
260 # We want these relative to the readout corner. If that's
261 # on the right side, we need to swap them.
262 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
263 ampStats['FIRST_MEAN'] = pixelZ
264 ampStats['LAST_MEAN'] = pixelA
265 else:
266 ampStats['FIRST_MEAN'] = pixelA
267 ampStats['LAST_MEAN'] = pixelZ
269 # Measure the columns of the overscan.
270 if overscans[ampIter] is None:
271 # The amplifier is likely entirely bad, and needs to
272 # be skipped.
273 self.log.warn("No overscan information available for ISR statistics for amp %s.",
274 amp.getName())
275 nCols = amp.getSerialOverscanBBox().getWidth()
276 ampStats['OVERSCAN_COLUMNS'] = np.full((nCols, ), np.nan)
277 ampStats['OVERSCAN_VALUES'] = np.full((nCols, ), np.nan)
278 else:
279 overscanImage = overscans[ampIter].overscanImage
280 columns = []
281 values = []
282 for column in range(0, overscanImage.getWidth()):
283 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column],
284 self.statType, self.statControl).getValue()
285 columns.append(column)
286 values.append(gain * osMean)
288 # We want these relative to the readout corner. If that's
289 # on the right side, we need to swap them.
290 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
291 ampStats['OVERSCAN_COLUMNS'] = list(reversed(columns))
292 ampStats['OVERSCAN_VALUES'] = list(reversed(values))
293 else:
294 ampStats['OVERSCAN_COLUMNS'] = columns
295 ampStats['OVERSCAN_VALUES'] = values
297 outputStats[amp.getName()] = ampStats
299 return outputStats
301 @staticmethod
302 def makeKernel(kernelSize):
303 """Make a boxcar smoothing kernel.
305 Parameters
306 ----------
307 kernelSize : `int`
308 Size of the kernel in pixels.
310 Returns
311 -------
312 kernel : `np.array`
313 Kernel for boxcar smoothing.
314 """
315 if kernelSize > 0:
316 kernel = np.full(kernelSize, 1.0 / kernelSize)
317 else:
318 kernel = np.array([1.0])
319 return kernel
321 def measureBanding(self, inputExp, overscans):
322 """Task to measure banding statistics.
324 Parameters
325 ----------
326 inputExp : `lsst.afw.image.Exposure`
327 Exposure to measure.
328 overscans : `list` [`lsst.pipe.base.Struct`]
329 List of overscan results. Expected fields are:
331 ``imageFit``
332 Value or fit subtracted from the amplifier image data
333 (scalar or `lsst.afw.image.Image`).
334 ``overscanFit``
335 Value or fit subtracted from the overscan image data
336 (scalar or `lsst.afw.image.Image`).
337 ``overscanImage``
338 Image of the overscan region with the overscan
339 correction applied (`lsst.afw.image.Image`). This
340 quantity is used to estimate the amplifier read noise
341 empirically.
343 Returns
344 -------
345 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
346 Dictionary of measurements, keyed by amplifier name and
347 statistics segment.
348 """
349 outputStats = {}
351 detector = inputExp.getDetector()
352 kernel = self.makeKernel(self.config.bandingKernelSize)
354 outputStats['AMP_BANDING'] = []
355 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
356 overscanFit = np.array(overscanData.overscanFit)
357 overscanArray = overscanData.overscanImage.image.array
358 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
360 smoothedOverscan = np.convolve(rawOverscan, kernel, mode='valid')
362 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
363 self.config.bandingFractionHigh])
364 outputStats['AMP_BANDING'].append(float(high - low))
366 if self.config.bandingUseHalfDetector:
367 fullLength = len(outputStats['AMP_BANDING'])
368 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING'][0:fullLength//2]))
369 else:
370 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING']))
372 return outputStats
374 def measureProjectionStatistics(self, inputExp, overscans):
375 """Task to measure metrics from image slicing.
377 Parameters
378 ----------
379 inputExp : `lsst.afw.image.Exposure`
380 Exposure to measure.
381 overscans : `list` [`lsst.pipe.base.Struct`]
382 List of overscan results. Expected fields are:
384 ``imageFit``
385 Value or fit subtracted from the amplifier image data
386 (scalar or `lsst.afw.image.Image`).
387 ``overscanFit``
388 Value or fit subtracted from the overscan image data
389 (scalar or `lsst.afw.image.Image`).
390 ``overscanImage``
391 Image of the overscan region with the overscan
392 correction applied (`lsst.afw.image.Image`). This
393 quantity is used to estimate the amplifier read noise
394 empirically.
396 Returns
397 -------
398 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
399 Dictionary of measurements, keyed by amplifier name and
400 statistics segment.
401 """
402 outputStats = {}
404 detector = inputExp.getDetector()
405 kernel = self.makeKernel(self.config.projectionKernelSize)
407 outputStats['AMP_VPROJECTION'] = {}
408 outputStats['AMP_HPROJECTION'] = {}
409 convolveMode = 'valid'
410 if self.config.doProjectionFft:
411 outputStats['AMP_VFFT_REAL'] = {}
412 outputStats['AMP_VFFT_IMAG'] = {}
413 outputStats['AMP_HFFT_REAL'] = {}
414 outputStats['AMP_HFFT_IMAG'] = {}
415 convolveMode = 'same'
417 for amp in detector.getAmplifiers():
418 ampArray = inputExp.image[amp.getBBox()].array
420 horizontalProjection = np.mean(ampArray, axis=0)
421 verticalProjection = np.mean(ampArray, axis=1)
423 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
424 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
426 outputStats['AMP_HPROJECTION'][amp.getName()] = horizontalProjection.tolist()
427 outputStats['AMP_VPROJECTION'][amp.getName()] = verticalProjection.tolist()
429 if self.config.doProjectionFft:
430 horizontalWindow = np.ones_like(horizontalProjection)
431 verticalWindow = np.ones_like(verticalProjection)
432 if self.config.projectionFftWindow == "NONE":
433 pass
434 elif self.config.projectionFftWindow == "HAMMING":
435 horizontalWindow = hamming(len(horizontalProjection))
436 verticalWindow = hamming(len(verticalProjection))
437 elif self.config.projectionFftWindow == "HANN":
438 horizontalWindow = hann(len(horizontalProjection))
439 verticalWindow = hann(len(verticalProjection))
440 elif self.config.projectionFftWindow == "GAUSSIAN":
441 horizontalWindow = gaussian(len(horizontalProjection))
442 verticalWindow = gaussian(len(verticalProjection))
443 else:
444 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
446 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
447 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
448 outputStats['AMP_HFFT_REAL'][amp.getName()] = np.real(horizontalFFT).tolist()
449 outputStats['AMP_HFFT_IMAG'][amp.getName()] = np.imag(horizontalFFT).tolist()
450 outputStats['AMP_VFFT_REAL'][amp.getName()] = np.real(verticalFFT).tolist()
451 outputStats['AMP_VFFT_IMAG'][amp.getName()] = np.imag(verticalFFT).tolist()
453 return outputStats