lsst.ip.isr gcc9f9eb5a9+f175ff0603
Loading...
Searching...
No Matches
isrStatistics.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
23
24import numpy as np
25
26from scipy.signal.windows import hamming, hann, gaussian
27from scipy.signal import butter, filtfilt
28from scipy.stats import linregress
29
30import lsst.afw.math as afwMath
31import lsst.afw.image as afwImage
32import lsst.pipe.base as pipeBase
33import lsst.pex.config as pexConfig
34
35from lsst.afw.cameraGeom import ReadoutCorner
36
37
38class IsrStatisticsTaskConfig(pexConfig.Config):
39 """Image statistics options.
40 """
41 doCtiStatistics = pexConfig.Field(
42 dtype=bool,
43 doc="Measure CTI statistics from image and overscans?",
44 default=False,
45 )
46 doApplyGainsForCtiStatistics = pexConfig.Field(
47 dtype=bool,
48 doc="Apply gain to the overscan region when measuring CTI statistics?",
49 default=True,
50 )
51
52 doBandingStatistics = pexConfig.Field(
53 dtype=bool,
54 doc="Measure image banding metric?",
55 default=False,
56 )
57 bandingKernelSize = pexConfig.Field(
58 dtype=int,
59 doc="Width of box for boxcar smoothing for banding metric.",
60 default=3,
61 check=lambda x: x == 0 or x % 2 != 0,
62 )
63 bandingFractionLow = pexConfig.Field(
64 dtype=float,
65 doc="Fraction of values to exclude from low samples.",
66 default=0.1,
67 check=lambda x: x >= 0.0 and x <= 1.0
68 )
69 bandingFractionHigh = pexConfig.Field(
70 dtype=float,
71 doc="Fraction of values to exclude from high samples.",
72 default=0.9,
73 check=lambda x: x >= 0.0 and x <= 1.0,
74 )
75 bandingUseHalfDetector = pexConfig.Field(
76 dtype=float,
77 doc="Use only the first half set of amplifiers.",
78 default=True,
79 )
80
81 doProjectionStatistics = pexConfig.Field(
82 dtype=bool,
83 doc="Measure projection metric?",
84 default=False,
85 )
86 projectionKernelSize = pexConfig.Field(
87 dtype=int,
88 doc="Width of box for boxcar smoothing of projections.",
89 default=0,
90 check=lambda x: x == 0 or x % 2 != 0,
91 )
92 doProjectionFft = pexConfig.Field(
93 dtype=bool,
94 doc="Generate FFTs from the image projections?",
95 default=False,
96 )
97 projectionFftWindow = pexConfig.ChoiceField(
98 dtype=str,
99 doc="Type of windowing to use prior to calculating FFT.",
100 default="HAMMING",
101 allowed={
102 "HAMMING": "Hamming window.",
103 "HANN": "Hann window.",
104 "GAUSSIAN": "Gaussian window.",
105 "NONE": "No window."
106 }
107 )
108
109 doCopyCalibDistributionStatistics = pexConfig.Field(
110 dtype=bool,
111 doc="Copy calibration distribution statistics to output?",
112 default=False,
113 )
114 expectedDistributionLevels = pexConfig.ListField(
115 dtype=float,
116 doc="Percentile levels expected in the calibration header.",
117 default=[0, 5, 16, 50, 84, 95, 100],
118 )
119
120 doBiasShiftStatistics = pexConfig.Field(
121 dtype=bool,
122 doc="Measure number of image shifts in overscan?",
123 default=False,
124 )
125 biasShiftFilterOrder = pexConfig.Field(
126 dtype=int,
127 doc="Filter order for Butterworth highpass filter.",
128 default=5,
129 )
130 biasShiftCutoff = pexConfig.Field(
131 dtype=float,
132 doc="Cutoff frequency for highpass filter.",
133 default=1.0/15.0,
134 )
135 biasShiftWindow = pexConfig.Field(
136 dtype=int,
137 doc="Filter window size in pixels for highpass filter.",
138 default=30,
139 )
140 biasShiftThreshold = pexConfig.Field(
141 dtype=float,
142 doc="S/N threshold for bias shift detection.",
143 default=3.0,
144 )
145 biasShiftRowSkip = pexConfig.Field(
146 dtype=int,
147 doc="Number of rows to skip for the bias shift detection.",
148 default=30,
149 )
150 biasShiftColumnSkip = pexConfig.Field(
151 dtype=int,
152 doc="Number of columns to skip when averaging the overscan region.",
153 default=3,
154 )
155
156 doAmplifierCorrelationStatistics = pexConfig.Field(
157 dtype=bool,
158 doc="Measure amplifier correlations?",
159 default=False,
160 )
161
162 stat = pexConfig.Field(
163 dtype=str,
164 default="MEANCLIP",
165 doc="Statistic name to use to measure regions.",
166 )
167 nSigmaClip = pexConfig.Field(
168 dtype=float,
169 default=3.0,
170 doc="Clipping threshold for background",
171 )
172 nIter = pexConfig.Field(
173 dtype=int,
174 default=3,
175 doc="Clipping iterations for background",
176 )
177 badMask = pexConfig.ListField(
178 dtype=str,
179 default=["BAD", "INTRP", "SAT"],
180 doc="Mask planes to ignore when identifying source pixels."
181 )
182
183
184class IsrStatisticsTask(pipeBase.Task):
185 """Task to measure arbitrary statistics on ISR processed exposures.
186
187 The goal is to wrap a number of optional measurements that are
188 useful for calibration production and detector stability.
189 """
190 ConfigClass = IsrStatisticsTaskConfig
191 _DefaultName = "isrStatistics"
192
193 def __init__(self, statControl=None, **kwargs):
194 super().__init__(**kwargs)
195 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
196 afwImage.Mask.getPlaneBitMask(self.config.badMask))
197 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
198
199 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
200 """Task to run arbitrary statistics.
201
202 The statistics should be measured by individual methods, and
203 add to the dictionary in the return struct.
204
205 Parameters
206 ----------
207 inputExp : `lsst.afw.image.Exposure`
208 The exposure to measure.
209 ptc : `lsst.ip.isr.PtcDataset`, optional
210 A PTC object containing gains to use.
211 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
212 List of overscan results. Expected fields are:
213
214 ``imageFit``
215 Value or fit subtracted from the amplifier image data
216 (scalar or `lsst.afw.image.Image`).
217 ``overscanFit``
218 Value or fit subtracted from the overscan image data
219 (scalar or `lsst.afw.image.Image`).
220 ``overscanImage``
221 Image of the overscan region with the overscan
222 correction applied (`lsst.afw.image.Image`). This
223 quantity is used to estimate the amplifier read noise
224 empirically.
225
226 Returns
227 -------
228 resultStruct : `lsst.pipe.base.Struct`
229 Contains the measured statistics as a dict stored in a
230 field named ``results``.
231
232 Raises
233 ------
234 RuntimeError
235 Raised if the amplifier gains could not be found.
236 """
237 # Find gains.
238 detector = inputExp.getDetector()
239 if ptc is not None:
240 gains = ptc.gain
241 elif detector is not None:
242 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
243 else:
244 raise RuntimeError("No source of gains provided.")
245
246 ctiResults = None
247 if self.config.doCtiStatistics:
248 ctiResults = self.measureCti(inputExp, overscanResults, gains)
249
250 bandingResults = None
251 if self.config.doBandingStatistics:
252 bandingResults = self.measureBanding(inputExp, overscanResults)
253
254 projectionResults = None
255 if self.config.doProjectionStatistics:
256 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
257
258 calibDistributionResults = None
259 if self.config.doCopyCalibDistributionStatistics:
260 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs)
261
262 biasShiftResults = None
263 if self.config.doBiasShiftStatistics:
264 biasShiftResults = self.measureBiasShifts(inputExp, overscanResults)
265
266 ampCorrelationResults = None
267 if self.config.doAmplifierCorrelationStatistics:
268 ampCorrelationResults = self.measureAmpCorrelations(inputExp, overscanResults)
269
270 mjd = inputExp.getMetadata().get("MJD", None)
271
272 return pipeBase.Struct(
273 results={"CTI": ctiResults,
274 "BANDING": bandingResults,
275 "PROJECTION": projectionResults,
276 "CALIBDIST": calibDistributionResults,
277 "BIASSHIFT": biasShiftResults,
278 "AMPCORR": ampCorrelationResults,
279 "MJD": mjd,
280 },
281 )
282
283 def measureCti(self, inputExp, overscans, gains):
284 """Task to measure CTI statistics.
285
286 Parameters
287 ----------
288 inputExp : `lsst.afw.image.Exposure`
289 Exposure to measure.
290 overscans : `list` [`lsst.pipe.base.Struct`]
291 List of overscan results. Expected fields are:
292
293 ``imageFit``
294 Value or fit subtracted from the amplifier image data
295 (scalar or `lsst.afw.image.Image`).
296 ``overscanFit``
297 Value or fit subtracted from the overscan image data
298 (scalar or `lsst.afw.image.Image`).
299 ``overscanImage``
300 Image of the overscan region with the overscan
301 correction applied (`lsst.afw.image.Image`). This
302 quantity is used to estimate the amplifier read noise
303 empirically.
304 gains : `dict` [`str` `float`]
305 Dictionary of per-amplifier gains, indexed by amplifier name.
306
307 Returns
308 -------
309 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
310 Dictionary of measurements, keyed by amplifier name and
311 statistics segment.
312 """
313 outputStats = {}
314
315 detector = inputExp.getDetector()
316 image = inputExp.image
317
318 # Ensure we have the same number of overscans as amplifiers.
319 assert len(overscans) == len(detector.getAmplifiers())
320
321 for ampIter, amp in enumerate(detector.getAmplifiers()):
322 ampStats = {}
323 gain = gains[amp.getName()]
324 readoutCorner = amp.getReadoutCorner()
325 # Full data region.
326 dataRegion = image[amp.getBBox()]
327 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType,
328 self.statControl).getValue()
329
330 # First and last image columns.
331 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
332 self.statType,
333 self.statControl).getValue()
334 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
335 self.statType,
336 self.statControl).getValue()
337
338 # We want these relative to the readout corner. If that's
339 # on the right side, we need to swap them.
340 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
341 ampStats["FIRST_MEAN"] = pixelZ
342 ampStats["LAST_MEAN"] = pixelA
343 else:
344 ampStats["FIRST_MEAN"] = pixelA
345 ampStats["LAST_MEAN"] = pixelZ
346
347 # Measure the columns of the overscan.
348 if overscans[ampIter] is None:
349 # The amplifier is likely entirely bad, and needs to
350 # be skipped.
351 self.log.warning("No overscan information available for ISR statistics for amp %s.",
352 amp.getName())
353 nCols = amp.getRawSerialOverscanBBox().getWidth()
354 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
355 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
356 else:
357 overscanImage = overscans[ampIter].overscanImage
358 columns = []
359 values = []
360 for column in range(0, overscanImage.getWidth()):
361 # If overscan.doParallelOverscan=True, the overscanImage
362 # will contain both the serial and parallel overscan
363 # regions.
364 # Only the serial overscan correction is implemented,
365 # so we must select only the serial overscan rows
366 # for a given column.
367 nRows = amp.getRawSerialOverscanBBox().getHeight()
368 osMean = afwMath.makeStatistics(overscanImage.image.array[:nRows, column],
369 self.statType, self.statControl).getValue()
370 columns.append(column)
371 if self.config.doApplyGainsForCtiStatistics:
372 values.append(gain * osMean)
373 else:
374 values.append(osMean)
375
376 # We want these relative to the readout corner. If that's
377 # on the right side, we need to swap them.
378 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
379 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns))
380 ampStats["OVERSCAN_VALUES"] = list(reversed(values))
381 else:
382 ampStats["OVERSCAN_COLUMNS"] = columns
383 ampStats["OVERSCAN_VALUES"] = values
384
385 outputStats[amp.getName()] = ampStats
386
387 return outputStats
388
389 @staticmethod
390 def makeKernel(kernelSize):
391 """Make a boxcar smoothing kernel.
392
393 Parameters
394 ----------
395 kernelSize : `int`
396 Size of the kernel in pixels.
397
398 Returns
399 -------
400 kernel : `np.array`
401 Kernel for boxcar smoothing.
402 """
403 if kernelSize > 0:
404 kernel = np.full(kernelSize, 1.0 / kernelSize)
405 else:
406 kernel = np.array([1.0])
407 return kernel
408
409 def measureBanding(self, inputExp, overscans):
410 """Task to measure banding statistics.
411
412 Parameters
413 ----------
414 inputExp : `lsst.afw.image.Exposure`
415 Exposure to measure.
416 overscans : `list` [`lsst.pipe.base.Struct`]
417 List of overscan results. Expected fields are:
418
419 ``imageFit``
420 Value or fit subtracted from the amplifier image data
421 (scalar or `lsst.afw.image.Image`).
422 ``overscanFit``
423 Value or fit subtracted from the overscan image data
424 (scalar or `lsst.afw.image.Image`).
425 ``overscanImage``
426 Image of the overscan region with the overscan
427 correction applied (`lsst.afw.image.Image`). This
428 quantity is used to estimate the amplifier read noise
429 empirically.
430
431 Returns
432 -------
433 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
434 Dictionary of measurements, keyed by amplifier name and
435 statistics segment.
436 """
437 outputStats = {}
438
439 detector = inputExp.getDetector()
440 kernel = self.makeKernel(self.config.bandingKernelSize)
441
442 outputStats["AMP_BANDING"] = []
443 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
444 overscanFit = np.array(overscanData.overscanFit)
445 overscanArray = overscanData.overscanImage.image.array
446 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
447
448 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid")
449
450 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
451 self.config.bandingFractionHigh])
452 outputStats["AMP_BANDING"].append(float(high - low))
453
454 if self.config.bandingUseHalfDetector:
455 fullLength = len(outputStats["AMP_BANDING"])
456 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2]))
457 else:
458 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"]))
459
460 return outputStats
461
462 def measureProjectionStatistics(self, inputExp, overscans):
463 """Task to measure metrics from image slicing.
464
465 Parameters
466 ----------
467 inputExp : `lsst.afw.image.Exposure`
468 Exposure to measure.
469 overscans : `list` [`lsst.pipe.base.Struct`]
470 List of overscan results. Expected fields are:
471
472 ``imageFit``
473 Value or fit subtracted from the amplifier image data
474 (scalar or `lsst.afw.image.Image`).
475 ``overscanFit``
476 Value or fit subtracted from the overscan image data
477 (scalar or `lsst.afw.image.Image`).
478 ``overscanImage``
479 Image of the overscan region with the overscan
480 correction applied (`lsst.afw.image.Image`). This
481 quantity is used to estimate the amplifier read noise
482 empirically.
483
484 Returns
485 -------
486 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
487 Dictionary of measurements, keyed by amplifier name and
488 statistics segment.
489 """
490 outputStats = {}
491
492 detector = inputExp.getDetector()
493 kernel = self.makeKernel(self.config.projectionKernelSize)
494
495 outputStats["AMP_VPROJECTION"] = {}
496 outputStats["AMP_HPROJECTION"] = {}
497 convolveMode = "valid"
498 if self.config.doProjectionFft:
499 outputStats["AMP_VFFT_REAL"] = {}
500 outputStats["AMP_VFFT_IMAG"] = {}
501 outputStats["AMP_HFFT_REAL"] = {}
502 outputStats["AMP_HFFT_IMAG"] = {}
503 convolveMode = "same"
504
505 for amp in detector.getAmplifiers():
506 ampArray = inputExp.image[amp.getBBox()].array
507
508 horizontalProjection = np.mean(ampArray, axis=0)
509 verticalProjection = np.mean(ampArray, axis=1)
510
511 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
512 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
513
514 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
515 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
516
517 if self.config.doProjectionFft:
518 horizontalWindow = np.ones_like(horizontalProjection)
519 verticalWindow = np.ones_like(verticalProjection)
520 if self.config.projectionFftWindow == "NONE":
521 pass
522 elif self.config.projectionFftWindow == "HAMMING":
523 horizontalWindow = hamming(len(horizontalProjection))
524 verticalWindow = hamming(len(verticalProjection))
525 elif self.config.projectionFftWindow == "HANN":
526 horizontalWindow = hann(len(horizontalProjection))
527 verticalWindow = hann(len(verticalProjection))
528 elif self.config.projectionFftWindow == "GAUSSIAN":
529 horizontalWindow = gaussian(len(horizontalProjection))
530 verticalWindow = gaussian(len(verticalProjection))
531 else:
532 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
533
534 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
535 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
536
537 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
538 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
539 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
540 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
541
542 return outputStats
543
544 def copyCalibDistributionStatistics(self, inputExp, **kwargs):
545 """Copy calibration statistics for this exposure.
546
547 Parameters
548 ----------
549 inputExp : `lsst.afw.image.Exposure`
550 The exposure being processed.
551 **kwargs :
552 Keyword arguments with calibrations.
553
554 Returns
555 -------
556 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
557 Dictionary of measurements, keyed by amplifier name and
558 statistics segment.
559 """
560 outputStats = {}
561
562 for amp in inputExp.getDetector():
563 ampStats = {}
564
565 for calibType in ("bias", "dark", "flat"):
566 if kwargs.get(calibType, None) is not None:
567 metadata = kwargs[calibType].getMetadata()
568 for pct in self.config.expectedDistributionLevels:
569 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
570 ampStats[key] = metadata.get(key, np.nan)
571 outputStats[amp.getName()] = ampStats
572 return outputStats
573
574 def measureBiasShifts(self, inputExp, overscanResults):
575 """Measure number of bias shifts from overscan data.
576
577 Parameters
578 ----------
579 inputExp : `lsst.afw.image.Exposure`
580 Exposure to measure.
581 overscans : `list` [`lsst.pipe.base.Struct`]
582 List of overscan results. Expected fields are:
583
584 ``imageFit``
585 Value or fit subtracted from the amplifier image data
586 (scalar or `lsst.afw.image.Image`).
587 ``overscanFit``
588 Value or fit subtracted from the overscan image data
589 (scalar or `lsst.afw.image.Image`).
590 ``overscanImage``
591 Image of the overscan region with the overscan
592 correction applied (`lsst.afw.image.Image`). This
593 quantity is used to estimate the amplifier read noise
594 empirically.
595
596 Returns
597 -------
598 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
599 Dictionary of measurements, keyed by amplifier name and
600 statistics segment.
601
602 Notes
603 -----
604 Based on eop_pipe implementation:
605 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505
606 """
607 outputStats = {}
608
609 detector = inputExp.getDetector()
610 for amp, overscans in zip(detector, overscanResults):
611 ampStats = {}
612 # Add fit back to data
613 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit
614
615 # Collapse array, skipping first three columns
616 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1)
617
618 # Scan for shifts
619 noise, shift_peaks = self._scan_for_shifts(rawOverscan)
620 ampStats["LOCAL_NOISE"] = float(noise)
621 ampStats["BIAS_SHIFTS"] = shift_peaks
622
623 outputStats[amp.getName()] = ampStats
624 return outputStats
625
626 def _scan_for_shifts(self, overscanData):
627 """Scan overscan data for shifts.
628
629 Parameters
630 ----------
631 overscanData : `list` [`float`]
632 Overscan data to search for shifts.
633
634 Returns
635 -------
636 noise : `float`
637 Noise estimated from Butterworth filtered overscan data.
638 peaks : `list` [`float`, `float`, `int`, `int`]
639 Shift peak information, containing the convolved peak
640 value, the raw peak value, and the lower and upper bounds
641 of the region checked.
642 """
643 numerator, denominator = butter(self.config.biasShiftFilterOrder,
644 self.config.biasShiftCutoff,
645 btype="high", analog=False)
646 noise = np.std(filtfilt(numerator, denominator, overscanData))
647 kernel = np.concatenate([np.arange(self.config.biasShiftWindow),
648 np.arange(-self.config.biasShiftWindow + 1, 0)])
649 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow])
650
651 convolved = np.convolve(overscanData, kernel, mode="valid")
652 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow))
653
654 shift_check = np.abs(convolved)/noise
655 shift_mask = shift_check > self.config.biasShiftThreshold
656 shift_mask[:self.config.biasShiftRowSkip] = False
657
658 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0),
659 shift_mask.view(np.int8),
660 np.int8(0)])).reshape(-1, 2)
661 shift_peaks = []
662 for region in shift_regions:
663 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0]
664 if self._satisfies_flatness(region_peak, convolved[region_peak], overscanData):
665 shift_peaks.append(
666 [float(convolved[region_peak]), float(region_peak),
667 int(region[0]), int(region[1])])
668 return noise, shift_peaks
669
670 def _satisfies_flatness(self, shiftRow, shiftPeak, overscanData):
671 """Determine if a region is flat.
672
673 Parameters
674 ----------
675 shiftRow : `int`
676 Row with possible peak.
677 shiftPeak : `float`
678 Value at the possible peak.
679 overscanData : `list` [`float`]
680 Overscan data used to fit around the possible peak.
681
682 Returns
683 -------
684 isFlat : `bool`
685 Indicates if the region is flat, and so the peak is valid.
686 """
687 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow)
688 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow)
689
690 preFit = linregress(prerange, overscanData[prerange])
691 postFit = linregress(postrange, overscanData[postrange])
692
693 if shiftPeak > 0:
694 preTrend = (2*preFit[0]*len(prerange) < shiftPeak)
695 postTrend = (2*postFit[0]*len(postrange) < shiftPeak)
696 else:
697 preTrend = (2*preFit[0]*len(prerange) > shiftPeak)
698 postTrend = (2*postFit[0]*len(postrange) > shiftPeak)
699
700 return (preTrend and postTrend)
701
702 def measureAmpCorrelations(self, inputExp, overscanResults):
703 """Measure correlations between amplifier segments.
704
705 Parameters
706 ----------
707 inputExp : `lsst.afw.image.Exposure`
708 Exposure to measure.
709 overscans : `list` [`lsst.pipe.base.Struct`]
710 List of overscan results. Expected fields are:
711
712 ``imageFit``
713 Value or fit subtracted from the amplifier image data
714 (scalar or `lsst.afw.image.Image`).
715 ``overscanFit``
716 Value or fit subtracted from the overscan image data
717 (scalar or `lsst.afw.image.Image`).
718 ``overscanImage``
719 Image of the overscan region with the overscan
720 correction applied (`lsst.afw.image.Image`). This
721 quantity is used to estimate the amplifier read noise
722 empirically.
723
724 Returns
725 -------
726 outputStats : `dict` [`str`, [`dict` [`str`,`float`]]
727 Dictionary of measurements, keyed by amplifier name and
728 statistics segment.
729
730 Notes
731 -----
732 Based on eo_pipe implementation:
733 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505
734 """
735 outputStats = {}
736
737 detector = inputExp.getDetector()
738
739 serialOSCorr = np.empty((len(detector), len(detector)))
740 imageCorr = np.empty((len(detector), len(detector)))
741 for ampId, overscan in enumerate(overscanResults):
742 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit
743 rawOverscan = rawOverscan.ravel()
744
745 ampImage = inputExp[detector[ampId].getBBox()]
746 ampImage = ampImage.image.array.ravel()
747
748 for ampId2, overscan2 in enumerate(overscanResults):
749
750 if ampId2 == ampId:
751 serialOSCorr[ampId, ampId2] = 1.0
752 imageCorr[ampId, ampId2] = 1.0
753 else:
754 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit
755 rawOverscan2 = rawOverscan2.ravel()
756
757 serialOSCorr[ampId, ampId2] = np.corrcoef(rawOverscan, rawOverscan2)[0, 1]
758
759 ampImage2 = inputExp[detector[ampId2].getBBox()]
760 ampImage2 = ampImage2.image.array.ravel()
761
762 imageCorr[ampId, ampId2] = np.corrcoef(ampImage, ampImage2)[0, 1]
763
764 outputStats["OVERSCAN_CORR"] = serialOSCorr.tolist()
765 outputStats["IMAGE_CORR"] = imageCorr.tolist()
766
767 return outputStats
measureCti(self, inputExp, overscans, gains)
copyCalibDistributionStatistics(self, inputExp, **kwargs)
measureAmpCorrelations(self, inputExp, overscanResults)
measureProjectionStatistics(self, inputExp, overscans)
run(self, inputExp, ptc=None, overscanResults=None, **kwargs)
_satisfies_flatness(self, shiftRow, shiftPeak, overscanData)
measureBiasShifts(self, inputExp, overscanResults)
__init__(self, statControl=None, **kwargs)