lsst.ip.isr g535a204a91+603d5f9333
Loading...
Searching...
No Matches
isrStatistics.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
23
24import numpy as np
25
26from scipy.signal.windows import hamming, hann, gaussian
27
28import lsst.afw.math as afwMath
29import lsst.afw.image as afwImage
30import lsst.pipe.base as pipeBase
31import lsst.pex.config as pexConfig
32
33from lsst.afw.cameraGeom import ReadoutCorner
34
35
36class IsrStatisticsTaskConfig(pexConfig.Config):
37 """Image statistics options.
38 """
39 doCtiStatistics = pexConfig.Field(
40 dtype=bool,
41 doc="Measure CTI statistics from image and overscans?",
42 default=False,
43 )
44 doApplyGainsForCtiStatistics = pexConfig.Field(
45 dtype=bool,
46 doc="Apply gain to the overscan region when measuring CTI statistics?",
47 default=True,
48 )
49
50 doBandingStatistics = pexConfig.Field(
51 dtype=bool,
52 doc="Measure image banding metric?",
53 default=False,
54 )
55 bandingKernelSize = pexConfig.Field(
56 dtype=int,
57 doc="Width of box for boxcar smoothing for banding metric.",
58 default=3,
59 check=lambda x: x == 0 or x % 2 != 0,
60 )
61 bandingFractionLow = pexConfig.Field(
62 dtype=float,
63 doc="Fraction of values to exclude from low samples.",
64 default=0.1,
65 check=lambda x: x >= 0.0 and x <= 1.0
66 )
67 bandingFractionHigh = pexConfig.Field(
68 dtype=float,
69 doc="Fraction of values to exclude from high samples.",
70 default=0.9,
71 check=lambda x: x >= 0.0 and x <= 1.0,
72 )
73 bandingUseHalfDetector = pexConfig.Field(
74 dtype=float,
75 doc="Use only the first half set of amplifiers.",
76 default=True,
77 )
78
79 doProjectionStatistics = pexConfig.Field(
80 dtype=bool,
81 doc="Measure projection metric?",
82 default=False,
83 )
84 projectionKernelSize = pexConfig.Field(
85 dtype=int,
86 doc="Width of box for boxcar smoothing of projections.",
87 default=0,
88 check=lambda x: x == 0 or x % 2 != 0,
89 )
90 doProjectionFft = pexConfig.Field(
91 dtype=bool,
92 doc="Generate FFTs from the image projections?",
93 default=False,
94 )
95 projectionFftWindow = pexConfig.ChoiceField(
96 dtype=str,
97 doc="Type of windowing to use prior to calculating FFT.",
98 default="HAMMING",
99 allowed={
100 "HAMMING": "Hamming window.",
101 "HANN": "Hann window.",
102 "GAUSSIAN": "Gaussian window.",
103 "NONE": "No window."
104 }
105 )
106
107 doCopyCalibDistributionStatistics = pexConfig.Field(
108 dtype=bool,
109 doc="Copy calibration distribution statistics to output?",
110 default=False,
111 )
112 expectedDistributionLevels = pexConfig.ListField(
113 dtype=float,
114 doc="Percentile levels expected in the calibration header.",
115 default=[0, 5, 16, 50, 84, 95, 100],
116 )
117
118 stat = pexConfig.Field(
119 dtype=str,
120 default="MEANCLIP",
121 doc="Statistic name to use to measure regions.",
122 )
123 nSigmaClip = pexConfig.Field(
124 dtype=float,
125 default=3.0,
126 doc="Clipping threshold for background",
127 )
128 nIter = pexConfig.Field(
129 dtype=int,
130 default=3,
131 doc="Clipping iterations for background",
132 )
133 badMask = pexConfig.ListField(
134 dtype=str,
135 default=["BAD", "INTRP", "SAT"],
136 doc="Mask planes to ignore when identifying source pixels."
137 )
138
139
140class IsrStatisticsTask(pipeBase.Task):
141 """Task to measure arbitrary statistics on ISR processed exposures.
142
143 The goal is to wrap a number of optional measurements that are
144 useful for calibration production and detector stability.
145 """
146 ConfigClass = IsrStatisticsTaskConfig
147 _DefaultName = "isrStatistics"
148
149 def __init__(self, statControl=None, **kwargs):
150 super().__init__(**kwargs)
151 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
152 afwImage.Mask.getPlaneBitMask(self.config.badMask))
153 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
154
155 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
156 """Task to run arbitrary statistics.
157
158 The statistics should be measured by individual methods, and
159 add to the dictionary in the return struct.
160
161 Parameters
162 ----------
163 inputExp : `lsst.afw.image.Exposure`
164 The exposure to measure.
165 ptc : `lsst.ip.isr.PtcDataset`, optional
166 A PTC object containing gains to use.
167 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
168 List of overscan results. Expected fields are:
169
170 ``imageFit``
171 Value or fit subtracted from the amplifier image data
172 (scalar or `lsst.afw.image.Image`).
173 ``overscanFit``
174 Value or fit subtracted from the overscan image data
175 (scalar or `lsst.afw.image.Image`).
176 ``overscanImage``
177 Image of the overscan region with the overscan
178 correction applied (`lsst.afw.image.Image`). This
179 quantity is used to estimate the amplifier read noise
180 empirically.
181
182 Returns
183 -------
184 resultStruct : `lsst.pipe.base.Struct`
185 Contains the measured statistics as a dict stored in a
186 field named ``results``.
187
188 Raises
189 ------
190 RuntimeError
191 Raised if the amplifier gains could not be found.
192 """
193 # Find gains.
194 detector = inputExp.getDetector()
195 if ptc is not None:
196 gains = ptc.gain
197 elif detector is not None:
198 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
199 else:
200 raise RuntimeError("No source of gains provided.")
201
202 ctiResults = None
203 if self.config.doCtiStatistics:
204 ctiResults = self.measureCti(inputExp, overscanResults, gains)
205
206 bandingResults = None
207 if self.config.doBandingStatistics:
208 bandingResults = self.measureBanding(inputExp, overscanResults)
209
210 projectionResults = None
211 if self.config.doProjectionStatistics:
212 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
213
214 calibDistributionResults = None
215 if self.config.doCopyCalibDistributionStatistics:
216 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs)
217
218 return pipeBase.Struct(
219 results={"CTI": ctiResults,
220 "BANDING": bandingResults,
221 "PROJECTION": projectionResults,
222 "CALIBDIST": calibDistributionResults,
223 },
224 )
225
226 def measureCti(self, inputExp, overscans, gains):
227 """Task to measure CTI statistics.
228
229 Parameters
230 ----------
231 inputExp : `lsst.afw.image.Exposure`
232 Exposure to measure.
233 overscans : `list` [`lsst.pipe.base.Struct`]
234 List of overscan results. Expected fields are:
235
236 ``imageFit``
237 Value or fit subtracted from the amplifier image data
238 (scalar or `lsst.afw.image.Image`).
239 ``overscanFit``
240 Value or fit subtracted from the overscan image data
241 (scalar or `lsst.afw.image.Image`).
242 ``overscanImage``
243 Image of the overscan region with the overscan
244 correction applied (`lsst.afw.image.Image`). This
245 quantity is used to estimate the amplifier read noise
246 empirically.
247 gains : `dict` [`str` `float`]
248 Dictionary of per-amplifier gains, indexed by amplifier name.
249
250 Returns
251 -------
252 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
253 Dictionary of measurements, keyed by amplifier name and
254 statistics segment.
255 """
256 outputStats = {}
257
258 detector = inputExp.getDetector()
259 image = inputExp.image
260
261 # Ensure we have the same number of overscans as amplifiers.
262 assert len(overscans) == len(detector.getAmplifiers())
263
264 for ampIter, amp in enumerate(detector.getAmplifiers()):
265 ampStats = {}
266 gain = gains[amp.getName()]
267 readoutCorner = amp.getReadoutCorner()
268 # Full data region.
269 dataRegion = image[amp.getBBox()]
270 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType,
271 self.statControl).getValue()
272
273 # First and last image columns.
274 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
275 self.statType,
276 self.statControl).getValue()
277 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
278 self.statType,
279 self.statControl).getValue()
280
281 # We want these relative to the readout corner. If that's
282 # on the right side, we need to swap them.
283 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
284 ampStats["FIRST_MEAN"] = pixelZ
285 ampStats["LAST_MEAN"] = pixelA
286 else:
287 ampStats["FIRST_MEAN"] = pixelA
288 ampStats["LAST_MEAN"] = pixelZ
289
290 # Measure the columns of the overscan.
291 if overscans[ampIter] is None:
292 # The amplifier is likely entirely bad, and needs to
293 # be skipped.
294 self.log.warning("No overscan information available for ISR statistics for amp %s.",
295 amp.getName())
296 nCols = amp.getSerialOverscanBBox().getWidth()
297 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
298 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
299 else:
300 overscanImage = overscans[ampIter].overscanImage
301 columns = []
302 values = []
303 for column in range(0, overscanImage.getWidth()):
304 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column],
305 self.statType, self.statControl).getValue()
306 columns.append(column)
307 if self.config.doApplyGainsForCtiStatistics:
308 values.append(gain * osMean)
309 else:
310 values.append(osMean)
311
312 # We want these relative to the readout corner. If that's
313 # on the right side, we need to swap them.
314 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
315 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns))
316 ampStats["OVERSCAN_VALUES"] = list(reversed(values))
317 else:
318 ampStats["OVERSCAN_COLUMNS"] = columns
319 ampStats["OVERSCAN_VALUES"] = values
320
321 outputStats[amp.getName()] = ampStats
322
323 return outputStats
324
325 @staticmethod
326 def makeKernel(kernelSize):
327 """Make a boxcar smoothing kernel.
328
329 Parameters
330 ----------
331 kernelSize : `int`
332 Size of the kernel in pixels.
333
334 Returns
335 -------
336 kernel : `np.array`
337 Kernel for boxcar smoothing.
338 """
339 if kernelSize > 0:
340 kernel = np.full(kernelSize, 1.0 / kernelSize)
341 else:
342 kernel = np.array([1.0])
343 return kernel
344
345 def measureBanding(self, inputExp, overscans):
346 """Task to measure banding statistics.
347
348 Parameters
349 ----------
350 inputExp : `lsst.afw.image.Exposure`
351 Exposure to measure.
352 overscans : `list` [`lsst.pipe.base.Struct`]
353 List of overscan results. Expected fields are:
354
355 ``imageFit``
356 Value or fit subtracted from the amplifier image data
357 (scalar or `lsst.afw.image.Image`).
358 ``overscanFit``
359 Value or fit subtracted from the overscan image data
360 (scalar or `lsst.afw.image.Image`).
361 ``overscanImage``
362 Image of the overscan region with the overscan
363 correction applied (`lsst.afw.image.Image`). This
364 quantity is used to estimate the amplifier read noise
365 empirically.
366
367 Returns
368 -------
369 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
370 Dictionary of measurements, keyed by amplifier name and
371 statistics segment.
372 """
373 outputStats = {}
374
375 detector = inputExp.getDetector()
376 kernel = self.makeKernel(self.config.bandingKernelSize)
377
378 outputStats["AMP_BANDING"] = []
379 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
380 overscanFit = np.array(overscanData.overscanFit)
381 overscanArray = overscanData.overscanImage.image.array
382 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
383
384 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid")
385
386 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
387 self.config.bandingFractionHigh])
388 outputStats["AMP_BANDING"].append(float(high - low))
389
390 if self.config.bandingUseHalfDetector:
391 fullLength = len(outputStats["AMP_BANDING"])
392 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2]))
393 else:
394 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"]))
395
396 return outputStats
397
398 def measureProjectionStatistics(self, inputExp, overscans):
399 """Task to measure metrics from image slicing.
400
401 Parameters
402 ----------
403 inputExp : `lsst.afw.image.Exposure`
404 Exposure to measure.
405 overscans : `list` [`lsst.pipe.base.Struct`]
406 List of overscan results. Expected fields are:
407
408 ``imageFit``
409 Value or fit subtracted from the amplifier image data
410 (scalar or `lsst.afw.image.Image`).
411 ``overscanFit``
412 Value or fit subtracted from the overscan image data
413 (scalar or `lsst.afw.image.Image`).
414 ``overscanImage``
415 Image of the overscan region with the overscan
416 correction applied (`lsst.afw.image.Image`). This
417 quantity is used to estimate the amplifier read noise
418 empirically.
419
420 Returns
421 -------
422 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
423 Dictionary of measurements, keyed by amplifier name and
424 statistics segment.
425 """
426 outputStats = {}
427
428 detector = inputExp.getDetector()
429 kernel = self.makeKernel(self.config.projectionKernelSize)
430
431 outputStats["AMP_VPROJECTION"] = {}
432 outputStats["AMP_HPROJECTION"] = {}
433 convolveMode = "valid"
434 if self.config.doProjectionFft:
435 outputStats["AMP_VFFT_REAL"] = {}
436 outputStats["AMP_VFFT_IMAG"] = {}
437 outputStats["AMP_HFFT_REAL"] = {}
438 outputStats["AMP_HFFT_IMAG"] = {}
439 convolveMode = "same"
440
441 for amp in detector.getAmplifiers():
442 ampArray = inputExp.image[amp.getBBox()].array
443
444 horizontalProjection = np.mean(ampArray, axis=0)
445 verticalProjection = np.mean(ampArray, axis=1)
446
447 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
448 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
449
450 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
451 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
452
453 if self.config.doProjectionFft:
454 horizontalWindow = np.ones_like(horizontalProjection)
455 verticalWindow = np.ones_like(verticalProjection)
456 if self.config.projectionFftWindow == "NONE":
457 pass
458 elif self.config.projectionFftWindow == "HAMMING":
459 horizontalWindow = hamming(len(horizontalProjection))
460 verticalWindow = hamming(len(verticalProjection))
461 elif self.config.projectionFftWindow == "HANN":
462 horizontalWindow = hann(len(horizontalProjection))
463 verticalWindow = hann(len(verticalProjection))
464 elif self.config.projectionFftWindow == "GAUSSIAN":
465 horizontalWindow = gaussian(len(horizontalProjection))
466 verticalWindow = gaussian(len(verticalProjection))
467 else:
468 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
469
470 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
471 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
472 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
473 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
474 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
475 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
476
477 return outputStats
478
479 def copyCalibDistributionStatistics(self, inputExp, **kwargs):
480 """Copy calibration statistics for this exposure.
481
482 Parameters
483 ----------
484 inputExp : `lsst.afw.image.Exposure`
485 The exposure being processed.
486 **kwargs :
487 Keyword arguments with calibrations.
488
489 Returns
490 -------
491 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
492 Dictionary of measurements, keyed by amplifier name and
493 statistics segment.
494 """
495 outputStats = {}
496
497 for amp in inputExp.getDetector():
498 ampStats = {}
499
500 for calibType in ("bias", "dark", "flat"):
501 if kwargs.get(calibType, None) is not None:
502 metadata = kwargs[calibType].getMetadata()
503 for pct in self.config.expectedDistributionLevels:
504 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
505 ampStats[key] = metadata.get(key, np.nan)
506 outputStats[amp.getName()] = ampStats
507
508 return outputStats
measureCti(self, inputExp, overscans, gains)
copyCalibDistributionStatistics(self, inputExp, **kwargs)
measureProjectionStatistics(self, inputExp, overscans)
run(self, inputExp, ptc=None, overscanResults=None, **kwargs)
__init__(self, statControl=None, **kwargs)