lsst.ip.isr ged8ae655b3+c95ea48ad7
Loading...
Searching...
No Matches
isrStatistics.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
23
24import numpy as np
25
26from scipy.signal.windows import hamming, hann, gaussian
27
28import lsst.afw.math as afwMath
29import lsst.afw.image as afwImage
30import lsst.pipe.base as pipeBase
31import lsst.pex.config as pexConfig
32
33from lsst.afw.cameraGeom import ReadoutCorner
34
35
36class IsrStatisticsTaskConfig(pexConfig.Config):
37 """Image statistics options.
38 """
39 doCtiStatistics = pexConfig.Field(
40 dtype=bool,
41 doc="Measure CTI statistics from image and overscans?",
42 default=False,
43 )
44 doApplyGainsForCtiStatistics = pexConfig.Field(
45 dtype=bool,
46 doc="Apply gain to the overscan region when measuring CTI statistics?",
47 default=True,
48 )
49
50 doBandingStatistics = pexConfig.Field(
51 dtype=bool,
52 doc="Measure image banding metric?",
53 default=False,
54 )
55 bandingKernelSize = pexConfig.Field(
56 dtype=int,
57 doc="Width of box for boxcar smoothing for banding metric.",
58 default=3,
59 check=lambda x: x == 0 or x % 2 != 0,
60 )
61 bandingFractionLow = pexConfig.Field(
62 dtype=float,
63 doc="Fraction of values to exclude from low samples.",
64 default=0.1,
65 check=lambda x: x >= 0.0 and x <= 1.0
66 )
67 bandingFractionHigh = pexConfig.Field(
68 dtype=float,
69 doc="Fraction of values to exclude from high samples.",
70 default=0.9,
71 check=lambda x: x >= 0.0 and x <= 1.0,
72 )
73 bandingUseHalfDetector = pexConfig.Field(
74 dtype=float,
75 doc="Use only the first half set of amplifiers.",
76 default=True,
77 )
78
79 doProjectionStatistics = pexConfig.Field(
80 dtype=bool,
81 doc="Measure projection metric?",
82 default=False,
83 )
84 projectionKernelSize = pexConfig.Field(
85 dtype=int,
86 doc="Width of box for boxcar smoothing of projections.",
87 default=0,
88 check=lambda x: x == 0 or x % 2 != 0,
89 )
90 doProjectionFft = pexConfig.Field(
91 dtype=bool,
92 doc="Generate FFTs from the image projections?",
93 default=False,
94 )
95 projectionFftWindow = pexConfig.ChoiceField(
96 dtype=str,
97 doc="Type of windowing to use prior to calculating FFT.",
98 default="HAMMING",
99 allowed={
100 "HAMMING": "Hamming window.",
101 "HANN": "Hann window.",
102 "GAUSSIAN": "Gaussian window.",
103 "NONE": "No window."
104 }
105 )
106
107 stat = pexConfig.Field(
108 dtype=str,
109 default='MEANCLIP',
110 doc="Statistic name to use to measure regions.",
111 )
112 nSigmaClip = pexConfig.Field(
113 dtype=float,
114 default=3.0,
115 doc="Clipping threshold for background",
116 )
117 nIter = pexConfig.Field(
118 dtype=int,
119 default=3,
120 doc="Clipping iterations for background",
121 )
122 badMask = pexConfig.ListField(
123 dtype=str,
124 default=["BAD", "INTRP", "SAT"],
125 doc="Mask planes to ignore when identifying source pixels."
126 )
127
128
129class IsrStatisticsTask(pipeBase.Task):
130 """Task to measure arbitrary statistics on ISR processed exposures.
131
132 The goal is to wrap a number of optional measurements that are
133 useful for calibration production and detector stability.
134 """
135 ConfigClass = IsrStatisticsTaskConfig
136 _DefaultName = "isrStatistics"
137
138 def __init__(self, statControl=None, **kwargs):
139 super().__init__(**kwargs)
140 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
141 afwImage.Mask.getPlaneBitMask(self.config.badMask))
142 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
143
144 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
145 """Task to run arbitrary statistics.
146
147 The statistics should be measured by individual methods, and
148 add to the dictionary in the return struct.
149
150 Parameters
151 ----------
152 inputExp : `lsst.afw.image.Exposure`
153 The exposure to measure.
154 ptc : `lsst.ip.isr.PtcDataset`, optional
155 A PTC object containing gains to use.
156 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
157 List of overscan results. Expected fields are:
158
159 ``imageFit``
160 Value or fit subtracted from the amplifier image data
161 (scalar or `lsst.afw.image.Image`).
162 ``overscanFit``
163 Value or fit subtracted from the overscan image data
164 (scalar or `lsst.afw.image.Image`).
165 ``overscanImage``
166 Image of the overscan region with the overscan
167 correction applied (`lsst.afw.image.Image`). This
168 quantity is used to estimate the amplifier read noise
169 empirically.
170
171 Returns
172 -------
173 resultStruct : `lsst.pipe.base.Struct`
174 Contains the measured statistics as a dict stored in a
175 field named ``results``.
176
177 Raises
178 ------
179 RuntimeError
180 Raised if the amplifier gains could not be found.
181 """
182 # Find gains.
183 detector = inputExp.getDetector()
184 if ptc is not None:
185 gains = ptc.gain
186 elif detector is not None:
187 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
188 else:
189 raise RuntimeError("No source of gains provided.")
190
191 ctiResults = None
192 if self.config.doCtiStatistics:
193 ctiResults = self.measureCti(inputExp, overscanResults, gains)
194
195 bandingResults = None
196 if self.config.doBandingStatistics:
197 bandingResults = self.measureBanding(inputExp, overscanResults)
198
199 projectionResults = None
200 if self.config.doProjectionStatistics:
201 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
202
203 return pipeBase.Struct(
204 results={'CTI': ctiResults,
205 'BANDING': bandingResults,
206 'PROJECTION': projectionResults,
207 },
208 )
209
210 def measureCti(self, inputExp, overscans, gains):
211 """Task to measure CTI statistics.
212
213 Parameters
214 ----------
215 inputExp : `lsst.afw.image.Exposure`
216 Exposure to measure.
217 overscans : `list` [`lsst.pipe.base.Struct`]
218 List of overscan results. Expected fields are:
219
220 ``imageFit``
221 Value or fit subtracted from the amplifier image data
222 (scalar or `lsst.afw.image.Image`).
223 ``overscanFit``
224 Value or fit subtracted from the overscan image data
225 (scalar or `lsst.afw.image.Image`).
226 ``overscanImage``
227 Image of the overscan region with the overscan
228 correction applied (`lsst.afw.image.Image`). This
229 quantity is used to estimate the amplifier read noise
230 empirically.
231 gains : `dict` [`str` `float`]
232 Dictionary of per-amplifier gains, indexed by amplifier name.
233
234 Returns
235 -------
236 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
237 Dictionary of measurements, keyed by amplifier name and
238 statistics segment.
239 """
240 outputStats = {}
241
242 detector = inputExp.getDetector()
243 image = inputExp.image
244
245 # Ensure we have the same number of overscans as amplifiers.
246 assert len(overscans) == len(detector.getAmplifiers())
247
248 for ampIter, amp in enumerate(detector.getAmplifiers()):
249 ampStats = {}
250 gain = gains[amp.getName()]
251 readoutCorner = amp.getReadoutCorner()
252 # Full data region.
253 dataRegion = image[amp.getBBox()]
254 ampStats['IMAGE_MEAN'] = afwMath.makeStatistics(dataRegion, self.statType,
255 self.statControl).getValue()
256
257 # First and last image columns.
258 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
259 self.statType,
260 self.statControl).getValue()
261 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
262 self.statType,
263 self.statControl).getValue()
264
265 # We want these relative to the readout corner. If that's
266 # on the right side, we need to swap them.
267 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
268 ampStats['FIRST_MEAN'] = pixelZ
269 ampStats['LAST_MEAN'] = pixelA
270 else:
271 ampStats['FIRST_MEAN'] = pixelA
272 ampStats['LAST_MEAN'] = pixelZ
273
274 # Measure the columns of the overscan.
275 if overscans[ampIter] is None:
276 # The amplifier is likely entirely bad, and needs to
277 # be skipped.
278 self.log.warn("No overscan information available for ISR statistics for amp %s.",
279 amp.getName())
280 nCols = amp.getSerialOverscanBBox().getWidth()
281 ampStats['OVERSCAN_COLUMNS'] = np.full((nCols, ), np.nan)
282 ampStats['OVERSCAN_VALUES'] = np.full((nCols, ), np.nan)
283 else:
284 overscanImage = overscans[ampIter].overscanImage
285 columns = []
286 values = []
287 for column in range(0, overscanImage.getWidth()):
288 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column],
289 self.statType, self.statControl).getValue()
290 columns.append(column)
291 if self.config.doApplyGainsForCtiStatistics:
292 values.append(gain * osMean)
293 else:
294 values.append(osMean)
295
296 # We want these relative to the readout corner. If that's
297 # on the right side, we need to swap them.
298 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
299 ampStats['OVERSCAN_COLUMNS'] = list(reversed(columns))
300 ampStats['OVERSCAN_VALUES'] = list(reversed(values))
301 else:
302 ampStats['OVERSCAN_COLUMNS'] = columns
303 ampStats['OVERSCAN_VALUES'] = values
304
305 outputStats[amp.getName()] = ampStats
306
307 return outputStats
308
309 @staticmethod
310 def makeKernel(kernelSize):
311 """Make a boxcar smoothing kernel.
312
313 Parameters
314 ----------
315 kernelSize : `int`
316 Size of the kernel in pixels.
317
318 Returns
319 -------
320 kernel : `np.array`
321 Kernel for boxcar smoothing.
322 """
323 if kernelSize > 0:
324 kernel = np.full(kernelSize, 1.0 / kernelSize)
325 else:
326 kernel = np.array([1.0])
327 return kernel
328
329 def measureBanding(self, inputExp, overscans):
330 """Task to measure banding statistics.
331
332 Parameters
333 ----------
334 inputExp : `lsst.afw.image.Exposure`
335 Exposure to measure.
336 overscans : `list` [`lsst.pipe.base.Struct`]
337 List of overscan results. Expected fields are:
338
339 ``imageFit``
340 Value or fit subtracted from the amplifier image data
341 (scalar or `lsst.afw.image.Image`).
342 ``overscanFit``
343 Value or fit subtracted from the overscan image data
344 (scalar or `lsst.afw.image.Image`).
345 ``overscanImage``
346 Image of the overscan region with the overscan
347 correction applied (`lsst.afw.image.Image`). This
348 quantity is used to estimate the amplifier read noise
349 empirically.
350
351 Returns
352 -------
353 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
354 Dictionary of measurements, keyed by amplifier name and
355 statistics segment.
356 """
357 outputStats = {}
358
359 detector = inputExp.getDetector()
360 kernel = self.makeKernel(self.config.bandingKernelSize)
361
362 outputStats['AMP_BANDING'] = []
363 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
364 overscanFit = np.array(overscanData.overscanFit)
365 overscanArray = overscanData.overscanImage.image.array
366 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
367
368 smoothedOverscan = np.convolve(rawOverscan, kernel, mode='valid')
369
370 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
371 self.config.bandingFractionHigh])
372 outputStats['AMP_BANDING'].append(float(high - low))
373
374 if self.config.bandingUseHalfDetector:
375 fullLength = len(outputStats['AMP_BANDING'])
376 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING'][0:fullLength//2]))
377 else:
378 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING']))
379
380 return outputStats
381
382 def measureProjectionStatistics(self, inputExp, overscans):
383 """Task to measure metrics from image slicing.
384
385 Parameters
386 ----------
387 inputExp : `lsst.afw.image.Exposure`
388 Exposure to measure.
389 overscans : `list` [`lsst.pipe.base.Struct`]
390 List of overscan results. Expected fields are:
391
392 ``imageFit``
393 Value or fit subtracted from the amplifier image data
394 (scalar or `lsst.afw.image.Image`).
395 ``overscanFit``
396 Value or fit subtracted from the overscan image data
397 (scalar or `lsst.afw.image.Image`).
398 ``overscanImage``
399 Image of the overscan region with the overscan
400 correction applied (`lsst.afw.image.Image`). This
401 quantity is used to estimate the amplifier read noise
402 empirically.
403
404 Returns
405 -------
406 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
407 Dictionary of measurements, keyed by amplifier name and
408 statistics segment.
409 """
410 outputStats = {}
411
412 detector = inputExp.getDetector()
413 kernel = self.makeKernel(self.config.projectionKernelSize)
414
415 outputStats['AMP_VPROJECTION'] = {}
416 outputStats['AMP_HPROJECTION'] = {}
417 convolveMode = 'valid'
418 if self.config.doProjectionFft:
419 outputStats['AMP_VFFT_REAL'] = {}
420 outputStats['AMP_VFFT_IMAG'] = {}
421 outputStats['AMP_HFFT_REAL'] = {}
422 outputStats['AMP_HFFT_IMAG'] = {}
423 convolveMode = 'same'
424
425 for amp in detector.getAmplifiers():
426 ampArray = inputExp.image[amp.getBBox()].array
427
428 horizontalProjection = np.mean(ampArray, axis=0)
429 verticalProjection = np.mean(ampArray, axis=1)
430
431 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
432 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
433
434 outputStats['AMP_HPROJECTION'][amp.getName()] = horizontalProjection.tolist()
435 outputStats['AMP_VPROJECTION'][amp.getName()] = verticalProjection.tolist()
436
437 if self.config.doProjectionFft:
438 horizontalWindow = np.ones_like(horizontalProjection)
439 verticalWindow = np.ones_like(verticalProjection)
440 if self.config.projectionFftWindow == "NONE":
441 pass
442 elif self.config.projectionFftWindow == "HAMMING":
443 horizontalWindow = hamming(len(horizontalProjection))
444 verticalWindow = hamming(len(verticalProjection))
445 elif self.config.projectionFftWindow == "HANN":
446 horizontalWindow = hann(len(horizontalProjection))
447 verticalWindow = hann(len(verticalProjection))
448 elif self.config.projectionFftWindow == "GAUSSIAN":
449 horizontalWindow = gaussian(len(horizontalProjection))
450 verticalWindow = gaussian(len(verticalProjection))
451 else:
452 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
453
454 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
455 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
456 outputStats['AMP_HFFT_REAL'][amp.getName()] = np.real(horizontalFFT).tolist()
457 outputStats['AMP_HFFT_IMAG'][amp.getName()] = np.imag(horizontalFFT).tolist()
458 outputStats['AMP_VFFT_REAL'][amp.getName()] = np.real(verticalFFT).tolist()
459 outputStats['AMP_VFFT_IMAG'][amp.getName()] = np.imag(verticalFFT).tolist()
460
461 return outputStats
measureCti(self, inputExp, overscans, gains)
measureProjectionStatistics(self, inputExp, overscans)
run(self, inputExp, ptc=None, overscanResults=None, **kwargs)
__init__(self, statControl=None, **kwargs)