Coverage for python/lsst/ip/isr/isrStatistics.py: 17%
251 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-03 11:38 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-03 11:38 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
24import numpy as np
26from scipy.signal.windows import hamming, hann, gaussian
27from scipy.signal import butter, filtfilt
28from scipy.stats import linregress
30import lsst.afw.math as afwMath
31import lsst.afw.image as afwImage
32import lsst.pipe.base as pipeBase
33import lsst.pex.config as pexConfig
35from lsst.afw.cameraGeom import ReadoutCorner
38class IsrStatisticsTaskConfig(pexConfig.Config):
39 """Image statistics options.
40 """
41 doCtiStatistics = pexConfig.Field(
42 dtype=bool,
43 doc="Measure CTI statistics from image and overscans?",
44 default=False,
45 )
46 doApplyGainsForCtiStatistics = pexConfig.Field(
47 dtype=bool,
48 doc="Apply gain to the overscan region when measuring CTI statistics?",
49 default=True,
50 )
52 doBandingStatistics = pexConfig.Field(
53 dtype=bool,
54 doc="Measure image banding metric?",
55 default=False,
56 )
57 bandingKernelSize = pexConfig.Field( 57 ↛ exitline 57 didn't jump to the function exit
58 dtype=int,
59 doc="Width of box for boxcar smoothing for banding metric.",
60 default=3,
61 check=lambda x: x == 0 or x % 2 != 0,
62 )
63 bandingFractionLow = pexConfig.Field( 63 ↛ exitline 63 didn't jump to the function exit
64 dtype=float,
65 doc="Fraction of values to exclude from low samples.",
66 default=0.1,
67 check=lambda x: x >= 0.0 and x <= 1.0
68 )
69 bandingFractionHigh = pexConfig.Field( 69 ↛ exitline 69 didn't jump to the function exit
70 dtype=float,
71 doc="Fraction of values to exclude from high samples.",
72 default=0.9,
73 check=lambda x: x >= 0.0 and x <= 1.0,
74 )
75 bandingUseHalfDetector = pexConfig.Field(
76 dtype=float,
77 doc="Use only the first half set of amplifiers.",
78 default=True,
79 )
81 doProjectionStatistics = pexConfig.Field(
82 dtype=bool,
83 doc="Measure projection metric?",
84 default=False,
85 )
86 projectionKernelSize = pexConfig.Field( 86 ↛ exitline 86 didn't jump to the function exit
87 dtype=int,
88 doc="Width of box for boxcar smoothing of projections.",
89 default=0,
90 check=lambda x: x == 0 or x % 2 != 0,
91 )
92 doProjectionFft = pexConfig.Field(
93 dtype=bool,
94 doc="Generate FFTs from the image projections?",
95 default=False,
96 )
97 projectionFftWindow = pexConfig.ChoiceField(
98 dtype=str,
99 doc="Type of windowing to use prior to calculating FFT.",
100 default="HAMMING",
101 allowed={
102 "HAMMING": "Hamming window.",
103 "HANN": "Hann window.",
104 "GAUSSIAN": "Gaussian window.",
105 "NONE": "No window."
106 }
107 )
109 doCopyCalibDistributionStatistics = pexConfig.Field(
110 dtype=bool,
111 doc="Copy calibration distribution statistics to output?",
112 default=False,
113 )
114 expectedDistributionLevels = pexConfig.ListField(
115 dtype=float,
116 doc="Percentile levels expected in the calibration header.",
117 default=[0, 5, 16, 50, 84, 95, 100],
118 )
120 doBiasShiftStatistics = pexConfig.Field(
121 dtype=bool,
122 doc="Measure number of image shifts in overscan?",
123 default=False,
124 )
125 biasShiftFilterOrder = pexConfig.Field(
126 dtype=int,
127 doc="Filter order for Butterworth highpass filter.",
128 default=5,
129 )
130 biasShiftCutoff = pexConfig.Field(
131 dtype=float,
132 doc="Cutoff frequency for highpass filter.",
133 default=1.0/15.0,
134 )
135 biasShiftWindow = pexConfig.Field(
136 dtype=int,
137 doc="Filter window size in pixels for highpass filter.",
138 default=30,
139 )
140 biasShiftThreshold = pexConfig.Field(
141 dtype=float,
142 doc="S/N threshold for bias shift detection.",
143 default=3.0,
144 )
145 biasShiftRowSkip = pexConfig.Field(
146 dtype=int,
147 doc="Number of rows to skip for the bias shift detection.",
148 default=30,
149 )
150 biasShiftColumnSkip = pexConfig.Field(
151 dtype=int,
152 doc="Number of columns to skip when averaging the overscan region.",
153 default=3,
154 )
156 doAmplifierCorrelationStatistics = pexConfig.Field(
157 dtype=bool,
158 doc="Measure amplifier correlations?",
159 default=False,
160 )
162 stat = pexConfig.Field(
163 dtype=str,
164 default="MEANCLIP",
165 doc="Statistic name to use to measure regions.",
166 )
167 nSigmaClip = pexConfig.Field(
168 dtype=float,
169 default=3.0,
170 doc="Clipping threshold for background",
171 )
172 nIter = pexConfig.Field(
173 dtype=int,
174 default=3,
175 doc="Clipping iterations for background",
176 )
177 badMask = pexConfig.ListField(
178 dtype=str,
179 default=["BAD", "INTRP", "SAT"],
180 doc="Mask planes to ignore when identifying source pixels."
181 )
184class IsrStatisticsTask(pipeBase.Task):
185 """Task to measure arbitrary statistics on ISR processed exposures.
187 The goal is to wrap a number of optional measurements that are
188 useful for calibration production and detector stability.
189 """
190 ConfigClass = IsrStatisticsTaskConfig
191 _DefaultName = "isrStatistics"
193 def __init__(self, statControl=None, **kwargs):
194 super().__init__(**kwargs)
195 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
196 afwImage.Mask.getPlaneBitMask(self.config.badMask))
197 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
199 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
200 """Task to run arbitrary statistics.
202 The statistics should be measured by individual methods, and
203 add to the dictionary in the return struct.
205 Parameters
206 ----------
207 inputExp : `lsst.afw.image.Exposure`
208 The exposure to measure.
209 ptc : `lsst.ip.isr.PtcDataset`, optional
210 A PTC object containing gains to use.
211 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
212 List of overscan results. Expected fields are:
214 ``imageFit``
215 Value or fit subtracted from the amplifier image data
216 (scalar or `lsst.afw.image.Image`).
217 ``overscanFit``
218 Value or fit subtracted from the overscan image data
219 (scalar or `lsst.afw.image.Image`).
220 ``overscanImage``
221 Image of the overscan region with the overscan
222 correction applied (`lsst.afw.image.Image`). This
223 quantity is used to estimate the amplifier read noise
224 empirically.
226 Returns
227 -------
228 resultStruct : `lsst.pipe.base.Struct`
229 Contains the measured statistics as a dict stored in a
230 field named ``results``.
232 Raises
233 ------
234 RuntimeError
235 Raised if the amplifier gains could not be found.
236 """
237 # Find gains.
238 detector = inputExp.getDetector()
239 if ptc is not None:
240 gains = ptc.gain
241 elif detector is not None:
242 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
243 else:
244 raise RuntimeError("No source of gains provided.")
246 ctiResults = None
247 if self.config.doCtiStatistics:
248 ctiResults = self.measureCti(inputExp, overscanResults, gains)
250 bandingResults = None
251 if self.config.doBandingStatistics:
252 bandingResults = self.measureBanding(inputExp, overscanResults)
254 projectionResults = None
255 if self.config.doProjectionStatistics:
256 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
258 calibDistributionResults = None
259 if self.config.doCopyCalibDistributionStatistics:
260 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs)
262 biasShiftResults = None
263 if self.config.doBiasShiftStatistics:
264 biasShiftResults = self.measureBiasShifts(inputExp, overscanResults)
266 ampCorrelationResults = None
267 if self.config.doAmplifierCorrelationStatistics:
268 ampCorrelationResults = self.measureAmpCorrelations(inputExp, overscanResults)
270 mjd = inputExp.getMetadata().get("MJD", None)
272 return pipeBase.Struct(
273 results={"CTI": ctiResults,
274 "BANDING": bandingResults,
275 "PROJECTION": projectionResults,
276 "CALIBDIST": calibDistributionResults,
277 "BIASSHIFT": biasShiftResults,
278 "AMPCORR": ampCorrelationResults,
279 "MJD": mjd,
280 },
281 )
283 def measureCti(self, inputExp, overscans, gains):
284 """Task to measure CTI statistics.
286 Parameters
287 ----------
288 inputExp : `lsst.afw.image.Exposure`
289 Exposure to measure.
290 overscans : `list` [`lsst.pipe.base.Struct`]
291 List of overscan results. Expected fields are:
293 ``imageFit``
294 Value or fit subtracted from the amplifier image data
295 (scalar or `lsst.afw.image.Image`).
296 ``overscanFit``
297 Value or fit subtracted from the overscan image data
298 (scalar or `lsst.afw.image.Image`).
299 ``overscanImage``
300 Image of the overscan region with the overscan
301 correction applied (`lsst.afw.image.Image`). This
302 quantity is used to estimate the amplifier read noise
303 empirically.
304 gains : `dict` [`str` `float`]
305 Dictionary of per-amplifier gains, indexed by amplifier name.
307 Returns
308 -------
309 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
310 Dictionary of measurements, keyed by amplifier name and
311 statistics segment.
312 """
313 outputStats = {}
315 detector = inputExp.getDetector()
316 image = inputExp.image
318 # Ensure we have the same number of overscans as amplifiers.
319 assert len(overscans) == len(detector.getAmplifiers())
321 for ampIter, amp in enumerate(detector.getAmplifiers()):
322 ampStats = {}
323 gain = gains[amp.getName()]
324 readoutCorner = amp.getReadoutCorner()
325 # Full data region.
326 dataRegion = image[amp.getBBox()]
327 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType,
328 self.statControl).getValue()
330 # First and last image columns.
331 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
332 self.statType,
333 self.statControl).getValue()
334 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
335 self.statType,
336 self.statControl).getValue()
338 # We want these relative to the readout corner. If that's
339 # on the right side, we need to swap them.
340 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
341 ampStats["FIRST_MEAN"] = pixelZ
342 ampStats["LAST_MEAN"] = pixelA
343 else:
344 ampStats["FIRST_MEAN"] = pixelA
345 ampStats["LAST_MEAN"] = pixelZ
347 # Measure the columns of the overscan.
348 if overscans[ampIter] is None:
349 # The amplifier is likely entirely bad, and needs to
350 # be skipped.
351 self.log.warning("No overscan information available for ISR statistics for amp %s.",
352 amp.getName())
353 nCols = amp.getRawSerialOverscanBBox().getWidth()
354 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
355 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
356 else:
357 overscanImage = overscans[ampIter].overscanImage
358 columns = []
359 values = []
360 for column in range(0, overscanImage.getWidth()):
361 # If overscan.doParallelOverscan=True, the overscanImage
362 # will contain both the serial and parallel overscan
363 # regions.
364 # Only the serial overscan correction is implemented,
365 # so we must select only the serial overscan rows
366 # for a given column.
367 nRows = amp.getRawSerialOverscanBBox().getHeight()
368 osMean = afwMath.makeStatistics(overscanImage.image.array[:nRows, column],
369 self.statType, self.statControl).getValue()
370 columns.append(column)
371 if self.config.doApplyGainsForCtiStatistics:
372 values.append(gain * osMean)
373 else:
374 values.append(osMean)
376 # We want these relative to the readout corner. If that's
377 # on the right side, we need to swap them.
378 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
379 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns))
380 ampStats["OVERSCAN_VALUES"] = list(reversed(values))
381 else:
382 ampStats["OVERSCAN_COLUMNS"] = columns
383 ampStats["OVERSCAN_VALUES"] = values
385 outputStats[amp.getName()] = ampStats
387 return outputStats
389 @staticmethod
390 def makeKernel(kernelSize):
391 """Make a boxcar smoothing kernel.
393 Parameters
394 ----------
395 kernelSize : `int`
396 Size of the kernel in pixels.
398 Returns
399 -------
400 kernel : `np.array`
401 Kernel for boxcar smoothing.
402 """
403 if kernelSize > 0:
404 kernel = np.full(kernelSize, 1.0 / kernelSize)
405 else:
406 kernel = np.array([1.0])
407 return kernel
409 def measureBanding(self, inputExp, overscans):
410 """Task to measure banding statistics.
412 Parameters
413 ----------
414 inputExp : `lsst.afw.image.Exposure`
415 Exposure to measure.
416 overscans : `list` [`lsst.pipe.base.Struct`]
417 List of overscan results. Expected fields are:
419 ``imageFit``
420 Value or fit subtracted from the amplifier image data
421 (scalar or `lsst.afw.image.Image`).
422 ``overscanFit``
423 Value or fit subtracted from the overscan image data
424 (scalar or `lsst.afw.image.Image`).
425 ``overscanImage``
426 Image of the overscan region with the overscan
427 correction applied (`lsst.afw.image.Image`). This
428 quantity is used to estimate the amplifier read noise
429 empirically.
431 Returns
432 -------
433 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
434 Dictionary of measurements, keyed by amplifier name and
435 statistics segment.
436 """
437 outputStats = {}
439 detector = inputExp.getDetector()
440 kernel = self.makeKernel(self.config.bandingKernelSize)
442 outputStats["AMP_BANDING"] = []
443 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
444 overscanFit = np.array(overscanData.overscanFit)
445 overscanArray = overscanData.overscanImage.image.array
446 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
448 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid")
450 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
451 self.config.bandingFractionHigh])
452 outputStats["AMP_BANDING"].append(float(high - low))
454 if self.config.bandingUseHalfDetector:
455 fullLength = len(outputStats["AMP_BANDING"])
456 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2]))
457 else:
458 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"]))
460 return outputStats
462 def measureProjectionStatistics(self, inputExp, overscans):
463 """Task to measure metrics from image slicing.
465 Parameters
466 ----------
467 inputExp : `lsst.afw.image.Exposure`
468 Exposure to measure.
469 overscans : `list` [`lsst.pipe.base.Struct`]
470 List of overscan results. Expected fields are:
472 ``imageFit``
473 Value or fit subtracted from the amplifier image data
474 (scalar or `lsst.afw.image.Image`).
475 ``overscanFit``
476 Value or fit subtracted from the overscan image data
477 (scalar or `lsst.afw.image.Image`).
478 ``overscanImage``
479 Image of the overscan region with the overscan
480 correction applied (`lsst.afw.image.Image`). This
481 quantity is used to estimate the amplifier read noise
482 empirically.
484 Returns
485 -------
486 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
487 Dictionary of measurements, keyed by amplifier name and
488 statistics segment.
489 """
490 outputStats = {}
492 detector = inputExp.getDetector()
493 kernel = self.makeKernel(self.config.projectionKernelSize)
495 outputStats["AMP_VPROJECTION"] = {}
496 outputStats["AMP_HPROJECTION"] = {}
497 convolveMode = "valid"
498 if self.config.doProjectionFft:
499 outputStats["AMP_VFFT_REAL"] = {}
500 outputStats["AMP_VFFT_IMAG"] = {}
501 outputStats["AMP_HFFT_REAL"] = {}
502 outputStats["AMP_HFFT_IMAG"] = {}
503 convolveMode = "same"
505 for amp in detector.getAmplifiers():
506 ampArray = inputExp.image[amp.getBBox()].array
508 horizontalProjection = np.mean(ampArray, axis=0)
509 verticalProjection = np.mean(ampArray, axis=1)
511 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
512 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
514 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
515 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
517 if self.config.doProjectionFft:
518 horizontalWindow = np.ones_like(horizontalProjection)
519 verticalWindow = np.ones_like(verticalProjection)
520 if self.config.projectionFftWindow == "NONE":
521 pass
522 elif self.config.projectionFftWindow == "HAMMING":
523 horizontalWindow = hamming(len(horizontalProjection))
524 verticalWindow = hamming(len(verticalProjection))
525 elif self.config.projectionFftWindow == "HANN":
526 horizontalWindow = hann(len(horizontalProjection))
527 verticalWindow = hann(len(verticalProjection))
528 elif self.config.projectionFftWindow == "GAUSSIAN":
529 horizontalWindow = gaussian(len(horizontalProjection))
530 verticalWindow = gaussian(len(verticalProjection))
531 else:
532 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
534 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
535 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
537 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
538 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
539 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
540 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
542 return outputStats
544 def copyCalibDistributionStatistics(self, inputExp, **kwargs):
545 """Copy calibration statistics for this exposure.
547 Parameters
548 ----------
549 inputExp : `lsst.afw.image.Exposure`
550 The exposure being processed.
551 **kwargs :
552 Keyword arguments with calibrations.
554 Returns
555 -------
556 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
557 Dictionary of measurements, keyed by amplifier name and
558 statistics segment.
559 """
560 outputStats = {}
562 for amp in inputExp.getDetector():
563 ampStats = {}
565 for calibType in ("bias", "dark", "flat"):
566 if kwargs.get(calibType, None) is not None:
567 metadata = kwargs[calibType].getMetadata()
568 for pct in self.config.expectedDistributionLevels:
569 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
570 ampStats[key] = metadata.get(key, np.nan)
571 outputStats[amp.getName()] = ampStats
572 return outputStats
574 def measureBiasShifts(self, inputExp, overscanResults):
575 """Measure number of bias shifts from overscan data.
577 Parameters
578 ----------
579 inputExp : `lsst.afw.image.Exposure`
580 Exposure to measure.
581 overscans : `list` [`lsst.pipe.base.Struct`]
582 List of overscan results. Expected fields are:
584 ``imageFit``
585 Value or fit subtracted from the amplifier image data
586 (scalar or `lsst.afw.image.Image`).
587 ``overscanFit``
588 Value or fit subtracted from the overscan image data
589 (scalar or `lsst.afw.image.Image`).
590 ``overscanImage``
591 Image of the overscan region with the overscan
592 correction applied (`lsst.afw.image.Image`). This
593 quantity is used to estimate the amplifier read noise
594 empirically.
596 Returns
597 -------
598 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
599 Dictionary of measurements, keyed by amplifier name and
600 statistics segment.
602 Notes
603 -----
604 Based on eop_pipe implementation:
605 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505
606 """
607 outputStats = {}
609 detector = inputExp.getDetector()
610 for amp, overscans in zip(detector, overscanResults):
611 ampStats = {}
612 # Add fit back to data
613 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit
615 # Collapse array, skipping first three columns
616 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1)
618 # Scan for shifts
619 noise, shift_peaks = self._scan_for_shifts(rawOverscan)
620 ampStats["LOCAL_NOISE"] = float(noise)
621 ampStats["BIAS_SHIFTS"] = shift_peaks
623 outputStats[amp.getName()] = ampStats
624 return outputStats
626 def _scan_for_shifts(self, overscanData):
627 """Scan overscan data for shifts.
629 Parameters
630 ----------
631 overscanData : `list` [`float`]
632 Overscan data to search for shifts.
634 Returns
635 -------
636 noise : `float`
637 Noise estimated from Butterworth filtered overscan data.
638 peaks : `list` [`float`, `float`, `int`, `int`]
639 Shift peak information, containing the convolved peak
640 value, the raw peak value, and the lower and upper bounds
641 of the region checked.
642 """
643 numerator, denominator = butter(self.config.biasShiftFilterOrder,
644 self.config.biasShiftCutoff,
645 btype="high", analog=False)
646 noise = np.std(filtfilt(numerator, denominator, overscanData))
647 kernel = np.concatenate([np.arange(self.config.biasShiftWindow),
648 np.arange(-self.config.biasShiftWindow + 1, 0)])
649 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow])
651 convolved = np.convolve(overscanData, kernel, mode="valid")
652 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow))
654 shift_check = np.abs(convolved)/noise
655 shift_mask = shift_check > self.config.biasShiftThreshold
656 shift_mask[:self.config.biasShiftRowSkip] = False
658 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0),
659 shift_mask.view(np.int8),
660 np.int8(0)])).reshape(-1, 2)
661 shift_peaks = []
662 for region in shift_regions:
663 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0]
664 if self._satisfies_flatness(region_peak, convolved[region_peak], overscanData):
665 shift_peaks.append(
666 [float(convolved[region_peak]), float(region_peak),
667 int(region[0]), int(region[1])])
668 return noise, shift_peaks
670 def _satisfies_flatness(self, shiftRow, shiftPeak, overscanData):
671 """Determine if a region is flat.
673 Parameters
674 ----------
675 shiftRow : `int`
676 Row with possible peak.
677 shiftPeak : `float`
678 Value at the possible peak.
679 overscanData : `list` [`float`]
680 Overscan data used to fit around the possible peak.
682 Returns
683 -------
684 isFlat : `bool`
685 Indicates if the region is flat, and so the peak is valid.
686 """
687 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow)
688 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow)
690 preFit = linregress(prerange, overscanData[prerange])
691 postFit = linregress(postrange, overscanData[postrange])
693 if shiftPeak > 0:
694 preTrend = (2*preFit[0]*len(prerange) < shiftPeak)
695 postTrend = (2*postFit[0]*len(postrange) < shiftPeak)
696 else:
697 preTrend = (2*preFit[0]*len(prerange) > shiftPeak)
698 postTrend = (2*postFit[0]*len(postrange) > shiftPeak)
700 return (preTrend and postTrend)
702 def measureAmpCorrelations(self, inputExp, overscanResults):
703 """Measure correlations between amplifier segments.
705 Parameters
706 ----------
707 inputExp : `lsst.afw.image.Exposure`
708 Exposure to measure.
709 overscans : `list` [`lsst.pipe.base.Struct`]
710 List of overscan results. Expected fields are:
712 ``imageFit``
713 Value or fit subtracted from the amplifier image data
714 (scalar or `lsst.afw.image.Image`).
715 ``overscanFit``
716 Value or fit subtracted from the overscan image data
717 (scalar or `lsst.afw.image.Image`).
718 ``overscanImage``
719 Image of the overscan region with the overscan
720 correction applied (`lsst.afw.image.Image`). This
721 quantity is used to estimate the amplifier read noise
722 empirically.
724 Returns
725 -------
726 outputStats : `dict` [`str`, [`dict` [`str`,`float`]]
727 Dictionary of measurements, keyed by amplifier name and
728 statistics segment.
730 Notes
731 -----
732 Based on eo_pipe implementation:
733 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505
734 """
735 outputStats = {}
737 detector = inputExp.getDetector()
739 serialOSCorr = np.empty((len(detector), len(detector)))
740 imageCorr = np.empty((len(detector), len(detector)))
741 for ampId, overscan in enumerate(overscanResults):
742 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit
743 rawOverscan = rawOverscan.ravel()
745 ampImage = inputExp[detector[ampId].getBBox()]
746 ampImage = ampImage.image.array.ravel()
748 for ampId2, overscan2 in enumerate(overscanResults):
750 if ampId2 == ampId:
751 serialOSCorr[ampId, ampId2] = 1.0
752 imageCorr[ampId, ampId2] = 1.0
753 else:
754 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit
755 rawOverscan2 = rawOverscan2.ravel()
757 serialOSCorr[ampId, ampId2] = np.corrcoef(rawOverscan, rawOverscan2)[0, 1]
759 ampImage2 = inputExp[detector[ampId2].getBBox()]
760 ampImage2 = ampImage2.image.array.ravel()
762 imageCorr[ampId, ampId2] = np.corrcoef(ampImage, ampImage2)[0, 1]
764 outputStats["OVERSCAN_CORR"] = serialOSCorr.tolist()
765 outputStats["IMAGE_CORR"] = imageCorr.tolist()
767 return outputStats