Coverage for python/lsst/ip/isr/isrStatistics.py: 15%
317 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-03 02:15 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-03 02:15 -0700
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"]
24import numpy as np
25import astropy.stats
26from scipy.signal.windows import hamming, hann, gaussian
27from scipy.signal import butter, filtfilt
28from scipy.stats import linregress
30import lsst.afw.math as afwMath
31import lsst.afw.image as afwImage
32import lsst.pipe.base as pipeBase
33import lsst.pex.config as pexConfig
35from lsst.afw.cameraGeom import ReadoutCorner
38class IsrStatisticsTaskConfig(pexConfig.Config):
39 """Image statistics options.
40 """
41 doCtiStatistics = pexConfig.Field(
42 dtype=bool,
43 doc="Measure CTI statistics from image and overscans?",
44 default=False,
45 )
46 doApplyGainsForCtiStatistics = pexConfig.Field(
47 dtype=bool,
48 doc="Apply gain to the overscan region when measuring CTI statistics?",
49 default=True,
50 )
52 doBandingStatistics = pexConfig.Field(
53 dtype=bool,
54 doc="Measure image banding metric?",
55 default=False,
56 )
57 bandingKernelSize = pexConfig.Field( 57 ↛ exitline 57 didn't jump to the function exit
58 dtype=int,
59 doc="Width of box for boxcar smoothing for banding metric.",
60 default=3,
61 check=lambda x: x == 0 or x % 2 != 0,
62 )
63 bandingFractionLow = pexConfig.Field( 63 ↛ exitline 63 didn't jump to the function exit
64 dtype=float,
65 doc="Fraction of values to exclude from low samples.",
66 default=0.1,
67 check=lambda x: x >= 0.0 and x <= 1.0
68 )
69 bandingFractionHigh = pexConfig.Field( 69 ↛ exitline 69 didn't jump to the function exit
70 dtype=float,
71 doc="Fraction of values to exclude from high samples.",
72 default=0.9,
73 check=lambda x: x >= 0.0 and x <= 1.0,
74 )
75 bandingUseHalfDetector = pexConfig.Field(
76 dtype=float,
77 doc="Use only the first half set of amplifiers.",
78 default=True,
79 )
81 doProjectionStatistics = pexConfig.Field(
82 dtype=bool,
83 doc="Measure projection metric?",
84 default=False,
85 )
86 projectionKernelSize = pexConfig.Field( 86 ↛ exitline 86 didn't jump to the function exit
87 dtype=int,
88 doc="Width of box for boxcar smoothing of projections.",
89 default=0,
90 check=lambda x: x == 0 or x % 2 != 0,
91 )
92 doProjectionFft = pexConfig.Field(
93 dtype=bool,
94 doc="Generate FFTs from the image projections?",
95 default=False,
96 )
97 projectionFftWindow = pexConfig.ChoiceField(
98 dtype=str,
99 doc="Type of windowing to use prior to calculating FFT.",
100 default="HAMMING",
101 allowed={
102 "HAMMING": "Hamming window.",
103 "HANN": "Hann window.",
104 "GAUSSIAN": "Gaussian window.",
105 "NONE": "No window."
106 }
107 )
108 doDivisaderoStatistics = pexConfig.Field(
109 dtype=bool,
110 doc="Measure divisadero tearing statistics?",
111 default=False,
112 )
113 divisaderoEdgePixels = pexConfig.Field(
114 dtype=int,
115 doc="Number of edge pixels excluded from divisadero linear fit.",
116 default=25,
117 )
118 divisaderoNumImpactPixels = pexConfig.Field(
119 dtype=int,
120 doc="Number of edge pixels to examine for divisadero tearing.",
121 default=2,
122 )
123 divisaderoProjectionMinimum = pexConfig.Field(
124 dtype=int,
125 doc="Minimum row to consider when taking robust mean of columns.",
126 default=10,
127 )
128 divisaderoProjectionMaximum = pexConfig.Field(
129 dtype=int,
130 doc="Maximum row to consider when taking robust mean of columns",
131 default=210,
132 )
133 doCopyCalibDistributionStatistics = pexConfig.Field(
134 dtype=bool,
135 doc="Copy calibration distribution statistics to output?",
136 default=False,
137 )
138 expectedDistributionLevels = pexConfig.ListField(
139 dtype=float,
140 doc="Percentile levels expected in the calibration header.",
141 default=[0, 5, 16, 50, 84, 95, 100],
142 )
144 doBiasShiftStatistics = pexConfig.Field(
145 dtype=bool,
146 doc="Measure number of image shifts in overscan?",
147 default=False,
148 )
149 biasShiftFilterOrder = pexConfig.Field(
150 dtype=int,
151 doc="Filter order for Butterworth highpass filter.",
152 default=5,
153 )
154 biasShiftCutoff = pexConfig.Field(
155 dtype=float,
156 doc="Cutoff frequency for highpass filter.",
157 default=1.0/15.0,
158 )
159 biasShiftWindow = pexConfig.Field(
160 dtype=int,
161 doc="Filter window size in pixels for highpass filter.",
162 default=30,
163 )
164 biasShiftThreshold = pexConfig.Field(
165 dtype=float,
166 doc="S/N threshold for bias shift detection.",
167 default=3.0,
168 )
169 biasShiftRowSkip = pexConfig.Field(
170 dtype=int,
171 doc="Number of rows to skip for the bias shift detection.",
172 default=30,
173 )
174 biasShiftColumnSkip = pexConfig.Field(
175 dtype=int,
176 doc="Number of columns to skip when averaging the overscan region.",
177 default=3,
178 )
180 doAmplifierCorrelationStatistics = pexConfig.Field(
181 dtype=bool,
182 doc="Measure amplifier correlations?",
183 default=False,
184 )
186 stat = pexConfig.Field(
187 dtype=str,
188 default="MEANCLIP",
189 doc="Statistic name to use to measure regions.",
190 )
191 nSigmaClip = pexConfig.Field(
192 dtype=float,
193 default=3.0,
194 doc="Clipping threshold for background",
195 )
196 nIter = pexConfig.Field(
197 dtype=int,
198 default=3,
199 doc="Clipping iterations for background",
200 )
201 badMask = pexConfig.ListField(
202 dtype=str,
203 default=["BAD", "INTRP", "SAT"],
204 doc="Mask planes to ignore when identifying source pixels."
205 )
208class IsrStatisticsTask(pipeBase.Task):
209 """Task to measure arbitrary statistics on ISR processed exposures.
211 The goal is to wrap a number of optional measurements that are
212 useful for calibration production and detector stability.
213 """
214 ConfigClass = IsrStatisticsTaskConfig
215 _DefaultName = "isrStatistics"
217 def __init__(self, statControl=None, **kwargs):
218 super().__init__(**kwargs)
219 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
220 afwImage.Mask.getPlaneBitMask(self.config.badMask))
221 self.statType = afwMath.stringToStatisticsProperty(self.config.stat)
223 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
224 """Task to run arbitrary statistics.
226 The statistics should be measured by individual methods, and
227 add to the dictionary in the return struct.
229 Parameters
230 ----------
231 inputExp : `lsst.afw.image.Exposure`
232 The exposure to measure.
233 ptc : `lsst.ip.isr.PtcDataset`, optional
234 A PTC object containing gains to use.
235 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
236 List of overscan results. Expected fields are:
238 ``imageFit``
239 Value or fit subtracted from the amplifier image data
240 (scalar or `lsst.afw.image.Image`).
241 ``overscanFit``
242 Value or fit subtracted from the overscan image data
243 (scalar or `lsst.afw.image.Image`).
244 ``overscanImage``
245 Image of the overscan region with the overscan
246 correction applied (`lsst.afw.image.Image`). This
247 quantity is used to estimate the amplifier read noise
248 empirically.
249 **kwargs :
250 Keyword arguments. Calibrations being passed in should
251 have an entry here.
253 Returns
254 -------
255 resultStruct : `lsst.pipe.base.Struct`
256 Contains the measured statistics as a dict stored in a
257 field named ``results``.
259 Raises
260 ------
261 RuntimeError
262 Raised if the amplifier gains could not be found.
263 """
264 # Find gains.
265 detector = inputExp.getDetector()
266 if ptc is not None:
267 gains = ptc.gain
268 elif detector is not None:
269 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()}
270 else:
271 raise RuntimeError("No source of gains provided.")
273 ctiResults = None
274 if self.config.doCtiStatistics:
275 ctiResults = self.measureCti(inputExp, overscanResults, gains)
277 bandingResults = None
278 if self.config.doBandingStatistics:
279 bandingResults = self.measureBanding(inputExp, overscanResults)
281 projectionResults = None
282 if self.config.doProjectionStatistics:
283 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults)
285 divisaderoResults = None
286 if self.config.doDivisaderoStatistics:
287 divisaderoResults = self.measureDivisaderoStatistics(inputExp, **kwargs)
289 calibDistributionResults = None
290 if self.config.doCopyCalibDistributionStatistics:
291 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs)
293 biasShiftResults = None
294 if self.config.doBiasShiftStatistics:
295 biasShiftResults = self.measureBiasShifts(inputExp, overscanResults)
297 ampCorrelationResults = None
298 if self.config.doAmplifierCorrelationStatistics:
299 ampCorrelationResults = self.measureAmpCorrelations(inputExp, overscanResults)
301 mjd = inputExp.getMetadata().get("MJD", None)
303 return pipeBase.Struct(
304 results={"CTI": ctiResults,
305 "BANDING": bandingResults,
306 "PROJECTION": projectionResults,
307 "CALIBDIST": calibDistributionResults,
308 "BIASSHIFT": biasShiftResults,
309 "AMPCORR": ampCorrelationResults,
310 "MJD": mjd,
311 'DIVISADERO': divisaderoResults,
312 },
313 )
315 def measureCti(self, inputExp, overscans, gains):
316 """Task to measure CTI statistics.
318 Parameters
319 ----------
320 inputExp : `lsst.afw.image.Exposure`
321 Exposure to measure.
322 overscans : `list` [`lsst.pipe.base.Struct`]
323 List of overscan results. Expected fields are:
325 ``imageFit``
326 Value or fit subtracted from the amplifier image data
327 (scalar or `lsst.afw.image.Image`).
328 ``overscanFit``
329 Value or fit subtracted from the overscan image data
330 (scalar or `lsst.afw.image.Image`).
331 ``overscanImage``
332 Image of the overscan region with the overscan
333 correction applied (`lsst.afw.image.Image`). This
334 quantity is used to estimate the amplifier read noise
335 empirically.
336 gains : `dict` [`str` `float`]
337 Dictionary of per-amplifier gains, indexed by amplifier name.
339 Returns
340 -------
341 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
342 Dictionary of measurements, keyed by amplifier name and
343 statistics segment.
344 """
345 outputStats = {}
347 detector = inputExp.getDetector()
348 image = inputExp.image
350 # Ensure we have the same number of overscans as amplifiers.
351 assert len(overscans) == len(detector.getAmplifiers())
353 for ampIter, amp in enumerate(detector.getAmplifiers()):
354 ampStats = {}
355 gain = gains[amp.getName()]
356 readoutCorner = amp.getReadoutCorner()
357 # Full data region.
358 dataRegion = image[amp.getBBox()]
359 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType,
360 self.statControl).getValue()
362 # First and last image columns.
363 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
364 self.statType,
365 self.statControl).getValue()
366 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
367 self.statType,
368 self.statControl).getValue()
370 # We want these relative to the readout corner. If that's
371 # on the right side, we need to swap them.
372 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
373 ampStats["FIRST_MEAN"] = pixelZ
374 ampStats["LAST_MEAN"] = pixelA
375 else:
376 ampStats["FIRST_MEAN"] = pixelA
377 ampStats["LAST_MEAN"] = pixelZ
379 # Measure the columns of the overscan.
380 if overscans[ampIter] is None:
381 # The amplifier is likely entirely bad, and needs to
382 # be skipped.
383 self.log.warning("No overscan information available for ISR statistics for amp %s.",
384 amp.getName())
385 nCols = amp.getRawSerialOverscanBBox().getWidth()
386 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
387 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
388 else:
389 overscanImage = overscans[ampIter].overscanImage
390 columns = []
391 values = []
392 for column in range(0, overscanImage.getWidth()):
393 # If overscan.doParallelOverscan=True, the overscanImage
394 # will contain both the serial and parallel overscan
395 # regions.
396 # Only the serial overscan correction is implemented,
397 # so we must select only the serial overscan rows
398 # for a given column.
399 nRows = amp.getRawSerialOverscanBBox().getHeight()
400 osMean = afwMath.makeStatistics(overscanImage.image.array[:nRows, column],
401 self.statType, self.statControl).getValue()
402 columns.append(column)
403 if self.config.doApplyGainsForCtiStatistics:
404 values.append(gain * osMean)
405 else:
406 values.append(osMean)
408 # We want these relative to the readout corner. If that's
409 # on the right side, we need to swap them.
410 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR):
411 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns))
412 ampStats["OVERSCAN_VALUES"] = list(reversed(values))
413 else:
414 ampStats["OVERSCAN_COLUMNS"] = columns
415 ampStats["OVERSCAN_VALUES"] = values
417 outputStats[amp.getName()] = ampStats
419 return outputStats
421 @staticmethod
422 def makeKernel(kernelSize):
423 """Make a boxcar smoothing kernel.
425 Parameters
426 ----------
427 kernelSize : `int`
428 Size of the kernel in pixels.
430 Returns
431 -------
432 kernel : `np.array`
433 Kernel for boxcar smoothing.
434 """
435 if kernelSize > 0:
436 kernel = np.full(kernelSize, 1.0 / kernelSize)
437 else:
438 kernel = np.array([1.0])
439 return kernel
441 def measureBanding(self, inputExp, overscans):
442 """Task to measure banding statistics.
444 Parameters
445 ----------
446 inputExp : `lsst.afw.image.Exposure`
447 Exposure to measure.
448 overscans : `list` [`lsst.pipe.base.Struct`]
449 List of overscan results. Expected fields are:
451 ``imageFit``
452 Value or fit subtracted from the amplifier image data
453 (scalar or `lsst.afw.image.Image`).
454 ``overscanFit``
455 Value or fit subtracted from the overscan image data
456 (scalar or `lsst.afw.image.Image`).
457 ``overscanImage``
458 Image of the overscan region with the overscan
459 correction applied (`lsst.afw.image.Image`). This
460 quantity is used to estimate the amplifier read noise
461 empirically.
463 Returns
464 -------
465 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
466 Dictionary of measurements, keyed by amplifier name and
467 statistics segment.
468 """
469 outputStats = {}
471 detector = inputExp.getDetector()
472 kernel = self.makeKernel(self.config.bandingKernelSize)
474 outputStats["AMP_BANDING"] = []
475 for amp, overscanData in zip(detector.getAmplifiers(), overscans):
476 overscanFit = np.array(overscanData.overscanFit)
477 overscanArray = overscanData.overscanImage.image.array
478 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
480 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid")
482 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
483 self.config.bandingFractionHigh])
484 outputStats["AMP_BANDING"].append(float(high - low))
486 if self.config.bandingUseHalfDetector:
487 fullLength = len(outputStats["AMP_BANDING"])
488 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2]))
489 else:
490 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"]))
492 return outputStats
494 def measureProjectionStatistics(self, inputExp, overscans):
495 """Task to measure metrics from image slicing.
497 Parameters
498 ----------
499 inputExp : `lsst.afw.image.Exposure`
500 Exposure to measure.
501 overscans : `list` [`lsst.pipe.base.Struct`]
502 List of overscan results. Expected fields are:
504 ``imageFit``
505 Value or fit subtracted from the amplifier image data
506 (scalar or `lsst.afw.image.Image`).
507 ``overscanFit``
508 Value or fit subtracted from the overscan image data
509 (scalar or `lsst.afw.image.Image`).
510 ``overscanImage``
511 Image of the overscan region with the overscan
512 correction applied (`lsst.afw.image.Image`). This
513 quantity is used to estimate the amplifier read noise
514 empirically.
516 Returns
517 -------
518 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
519 Dictionary of measurements, keyed by amplifier name and
520 statistics segment.
521 """
522 outputStats = {}
524 detector = inputExp.getDetector()
525 kernel = self.makeKernel(self.config.projectionKernelSize)
527 outputStats["AMP_VPROJECTION"] = {}
528 outputStats["AMP_HPROJECTION"] = {}
529 convolveMode = "valid"
530 if self.config.doProjectionFft:
531 outputStats["AMP_VFFT_REAL"] = {}
532 outputStats["AMP_VFFT_IMAG"] = {}
533 outputStats["AMP_HFFT_REAL"] = {}
534 outputStats["AMP_HFFT_IMAG"] = {}
535 convolveMode = "same"
537 for amp in detector.getAmplifiers():
538 ampArray = inputExp.image[amp.getBBox()].array
540 horizontalProjection = np.mean(ampArray, axis=0)
541 verticalProjection = np.mean(ampArray, axis=1)
543 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
544 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
546 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
547 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
549 if self.config.doProjectionFft:
550 horizontalWindow = np.ones_like(horizontalProjection)
551 verticalWindow = np.ones_like(verticalProjection)
552 if self.config.projectionFftWindow == "NONE":
553 pass
554 elif self.config.projectionFftWindow == "HAMMING":
555 horizontalWindow = hamming(len(horizontalProjection))
556 verticalWindow = hamming(len(verticalProjection))
557 elif self.config.projectionFftWindow == "HANN":
558 horizontalWindow = hann(len(horizontalProjection))
559 verticalWindow = hann(len(verticalProjection))
560 elif self.config.projectionFftWindow == "GAUSSIAN":
561 horizontalWindow = gaussian(len(horizontalProjection))
562 verticalWindow = gaussian(len(verticalProjection))
563 else:
564 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}")
566 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
567 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
569 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
570 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
571 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
572 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
574 return outputStats
576 def copyCalibDistributionStatistics(self, inputExp, **kwargs):
577 """Copy calibration statistics for this exposure.
579 Parameters
580 ----------
581 inputExp : `lsst.afw.image.Exposure`
582 The exposure being processed.
583 **kwargs :
584 Keyword arguments with calibrations.
586 Returns
587 -------
588 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
589 Dictionary of measurements, keyed by amplifier name and
590 statistics segment.
591 """
592 outputStats = {}
594 # Amp level elements
595 for amp in inputExp.getDetector():
596 ampStats = {}
598 for calibType in ("bias", "dark", "flat"):
599 if kwargs.get(calibType, None) is not None:
600 metadata = kwargs[calibType].getMetadata()
601 for pct in self.config.expectedDistributionLevels:
602 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
603 ampStats[key] = metadata.get(key, np.nan)
605 for calibType in ("defects"):
606 if kwargs.get(calibType, None) is not None:
607 metadata = kwargs[calibType].getMetadata()
608 for key in (f"LSST CALIB {calibType.upper()} {amp.getName()} N_HOT",
609 f"LSST CALIB {calibType.upper()} {amp.getName()} N_COLD"):
610 ampStats[key] = metadata.get(key, np.nan)
611 outputStats[amp.getName()] = ampStats
613 # Detector level elements
614 for calibType in ("defects"):
615 if kwargs.get(calibType, None) is not None:
616 metadata = kwargs[calibType].getMetadata()
617 for key in (f"LSST CALIB {calibType.upper()} N_BAD_COLUMNS"):
618 outputStats["detector"][key] = metadata.get(key, np.nan)
620 return outputStats
622 def measureBiasShifts(self, inputExp, overscanResults):
623 """Measure number of bias shifts from overscan data.
625 Parameters
626 ----------
627 inputExp : `lsst.afw.image.Exposure`
628 Exposure to measure.
629 overscans : `list` [`lsst.pipe.base.Struct`]
630 List of overscan results. Expected fields are:
632 ``imageFit``
633 Value or fit subtracted from the amplifier image data
634 (scalar or `lsst.afw.image.Image`).
635 ``overscanFit``
636 Value or fit subtracted from the overscan image data
637 (scalar or `lsst.afw.image.Image`).
638 ``overscanImage``
639 Image of the overscan region with the overscan
640 correction applied (`lsst.afw.image.Image`). This
641 quantity is used to estimate the amplifier read noise
642 empirically.
644 Returns
645 -------
646 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
647 Dictionary of measurements, keyed by amplifier name and
648 statistics segment.
650 Notes
651 -----
652 Based on eop_pipe implementation:
653 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505
654 """
655 outputStats = {}
657 detector = inputExp.getDetector()
658 for amp, overscans in zip(detector, overscanResults):
659 ampStats = {}
660 # Add fit back to data
661 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit
663 # Collapse array, skipping first three columns
664 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1)
666 # Scan for shifts
667 noise, shift_peaks = self._scan_for_shifts(rawOverscan)
668 ampStats["LOCAL_NOISE"] = float(noise)
669 ampStats["BIAS_SHIFTS"] = shift_peaks
671 outputStats[amp.getName()] = ampStats
672 return outputStats
674 def _scan_for_shifts(self, overscanData):
675 """Scan overscan data for shifts.
677 Parameters
678 ----------
679 overscanData : `list` [`float`]
680 Overscan data to search for shifts.
682 Returns
683 -------
684 noise : `float`
685 Noise estimated from Butterworth filtered overscan data.
686 peaks : `list` [`float`, `float`, `int`, `int`]
687 Shift peak information, containing the convolved peak
688 value, the raw peak value, and the lower and upper bounds
689 of the region checked.
690 """
691 numerator, denominator = butter(self.config.biasShiftFilterOrder,
692 self.config.biasShiftCutoff,
693 btype="high", analog=False)
694 noise = np.std(filtfilt(numerator, denominator, overscanData))
695 kernel = np.concatenate([np.arange(self.config.biasShiftWindow),
696 np.arange(-self.config.biasShiftWindow + 1, 0)])
697 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow])
699 convolved = np.convolve(overscanData, kernel, mode="valid")
700 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow))
702 shift_check = np.abs(convolved)/noise
703 shift_mask = shift_check > self.config.biasShiftThreshold
704 shift_mask[:self.config.biasShiftRowSkip] = False
706 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0),
707 shift_mask.view(np.int8),
708 np.int8(0)])).reshape(-1, 2)
709 shift_peaks = []
710 for region in shift_regions:
711 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0]
712 if self._satisfies_flatness(region_peak, convolved[region_peak], overscanData):
713 shift_peaks.append(
714 [float(convolved[region_peak]), float(region_peak),
715 int(region[0]), int(region[1])])
716 return noise, shift_peaks
718 def _satisfies_flatness(self, shiftRow, shiftPeak, overscanData):
719 """Determine if a region is flat.
721 Parameters
722 ----------
723 shiftRow : `int`
724 Row with possible peak.
725 shiftPeak : `float`
726 Value at the possible peak.
727 overscanData : `list` [`float`]
728 Overscan data used to fit around the possible peak.
730 Returns
731 -------
732 isFlat : `bool`
733 Indicates if the region is flat, and so the peak is valid.
734 """
735 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow)
736 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow)
738 preFit = linregress(prerange, overscanData[prerange])
739 postFit = linregress(postrange, overscanData[postrange])
741 if shiftPeak > 0:
742 preTrend = (2*preFit[0]*len(prerange) < shiftPeak)
743 postTrend = (2*postFit[0]*len(postrange) < shiftPeak)
744 else:
745 preTrend = (2*preFit[0]*len(prerange) > shiftPeak)
746 postTrend = (2*postFit[0]*len(postrange) > shiftPeak)
748 return (preTrend and postTrend)
750 def measureAmpCorrelations(self, inputExp, overscanResults):
751 """Measure correlations between amplifier segments.
753 Parameters
754 ----------
755 inputExp : `lsst.afw.image.Exposure`
756 Exposure to measure.
757 overscans : `list` [`lsst.pipe.base.Struct`]
758 List of overscan results. Expected fields are:
760 ``imageFit``
761 Value or fit subtracted from the amplifier image data
762 (scalar or `lsst.afw.image.Image`).
763 ``overscanFit``
764 Value or fit subtracted from the overscan image data
765 (scalar or `lsst.afw.image.Image`).
766 ``overscanImage``
767 Image of the overscan region with the overscan
768 correction applied (`lsst.afw.image.Image`). This
769 quantity is used to estimate the amplifier read noise
770 empirically.
772 Returns
773 -------
774 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
775 Dictionary of measurements, keyed by amplifier name and
776 statistics segment.
778 Notes
779 -----
780 Based on eo_pipe implementation:
781 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505
782 """
783 detector = inputExp.getDetector()
784 rows = []
786 for ampId, overscan in enumerate(overscanResults):
787 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit
788 rawOverscan = rawOverscan.ravel()
790 ampImage = inputExp[detector[ampId].getBBox()]
791 ampImage = ampImage.image.array.ravel()
793 for ampId2, overscan2 in enumerate(overscanResults):
794 osC = 1.0
795 imC = 1.0
796 if ampId2 != ampId:
797 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit
798 rawOverscan2 = rawOverscan2.ravel()
800 osC = np.corrcoef(rawOverscan, rawOverscan2)[0, 1]
802 ampImage2 = inputExp[detector[ampId2].getBBox()]
803 ampImage2 = ampImage2.image.array.ravel()
805 imC = np.corrcoef(ampImage, ampImage2)[0, 1]
806 rows.append(
807 {'detector': detector.getId(),
808 'detectorComp': detector.getId(),
809 'ampName': detector[ampId].getName(),
810 'ampComp': detector[ampId2].getName(),
811 'imageCorr': float(imC),
812 'overscanCorr': float(osC),
813 }
814 )
815 return rows
817 def measureDivisaderoStatistics(self, inputExp, **kwargs):
818 """Measure Max Divisadero Tearing effect per amp.
820 Parameters
821 ----------
822 inputExp : `lsst.afw.image.Exposure`
823 Exposure to measure. Usually a flat.
824 **kwargs :
825 The flat will be selected from here.
827 Returns
828 -------
829 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
830 Dictionary of measurements, keyed by amplifier name and
831 statistics segment.
832 Measurements include
834 - DIVISADERO_PROFILE: Robust mean of rows between
835 divisaderoProjection<Maximum|Minumum> on readout edge of ccd
836 normalized by a linear fit to the same rows.
837 - DIVISADERO_MAX_PAIR: Tuple of maximum of the absolute values of
838 the DIVISADERO_PROFILE, for number of pixels (specified by
839 divisaderoNumImpactPixels on left and right side of amp.
840 - DIVISADERO_MAX: Maximum of the absolute values of the
841 the DIVISADERO_PROFILE, for the divisaderoNumImpactPixels on
842 boundaries of neighboring amps (including the pixels in those
843 neighborboring amps).
844 """
845 outputStats = {}
847 for amp in inputExp.getDetector():
848 # Copy unneeded if we do not ever modify the array by flipping
849 ampArray = inputExp.image[amp.getBBox()].array
850 # slice the outer top or bottom of the amp: the readout side
851 if amp.getReadoutCorner().name in ('UL', 'UR'):
852 minRow = amp.getBBox().getHeight() - self.config.divisaderoProjectionMaximum
853 maxRow = amp.getBBox().getHeight() - self.config.divisaderoProjectionMinimum
854 else:
855 minRow = self.config.divisaderoProjectionMinimum
856 maxRow = self.config.divisaderoProjectionMaximum
858 segment = slice(minRow, maxRow)
859 projection, _, _ = astropy.stats.sigma_clipped_stats(ampArray[segment, :], axis=0)
861 ampStats = {}
862 projection /= np.median(projection)
863 columns = np.arange(len(projection))
865 segment = slice(self.config.divisaderoEdgePixels, -self.config.divisaderoEdgePixels)
866 model = np.polyfit(columns[segment], projection[segment], 1)
867 modelProjection = model[0] * columns + model[1]
868 divisaderoProfile = projection / modelProjection
870 # look for max at the edges:
871 leftMax = np.nanmax(np.abs(divisaderoProfile[0:self.config.divisaderoNumImpactPixels] - 1.0))
872 rightMax = np.nanmax(np.abs(divisaderoProfile[-self.config.divisaderoNumImpactPixels:] - 1.0))
874 ampStats['DIVISADERO_PROFILE'] = np.array(divisaderoProfile).tolist()
875 ampStats['DIVISADERO_MAX_PAIR'] = [leftMax, rightMax]
876 outputStats[amp.getName()] = ampStats
878 detector = inputExp.getDetector()
879 xCenters = [amp.getBBox().getCenterX() for amp in detector]
880 yCenters = [amp.getBBox().getCenterY() for amp in detector]
881 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1
882 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1
883 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int)
884 for ampId, xIndex, yIndex in zip(np.arange(len(detector)), xIndices, yIndices):
885 ampIds[yIndex, xIndex] = ampId
887 # Loop over amps again because the DIVISIDERO_MAX will be the max
888 # of the profile on its boundary with its neighboring amps
889 for i, amp in enumerate(detector):
890 y, x = np.where(ampIds == i)
891 end = ampIds.shape[1] - 1
892 xInd = x[0]
893 yInd = y[0]
894 thisAmpsPair = outputStats[amp.getName()]['DIVISADERO_MAX_PAIR']
896 if x == 0:
897 # leftmost amp: take the max of your right side and
898 myMax = thisAmpsPair[1]
899 # your neighbor's left side
900 neighborMax = outputStats[detector[ampIds[yInd, 1]].getName()]['DIVISADERO_MAX_PAIR'][0]
901 elif x == end:
902 # rightmost amp: take the max of your left side and
903 myMax = thisAmpsPair[0]
904 # your neighbor's right side
905 neighborMax = outputStats[detector[ampIds[yInd, end - 1]].getName()]['DIVISADERO_MAX_PAIR'][1]
906 else:
907 # Middle amp: take the max of both your own sides and the
908 myMax = max(thisAmpsPair)
909 leftName = detector[ampIds[yInd, max(xInd - 1, 0)]].getName()
910 rightName = detector[ampIds[yInd, min(xInd + 1, ampIds.shape[1] - 1)]].getName()
911 # right side of the neighbor to your left
912 # and left side of your neighbor to your right
913 neighborMax = max(outputStats[leftName]['DIVISADERO_MAX_PAIR'][1],
914 outputStats[rightName]['DIVISADERO_MAX_PAIR'][0])
916 divisaderoMax = max([myMax, neighborMax])
917 outputStats[amp.getName()]['DIVISADERO_MAX'] = divisaderoMax
919 return outputStats