195 self.
statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
196 afwImage.Mask.getPlaneBitMask(self.config.badMask))
197 self.
statType = afwMath.stringToStatisticsProperty(self.config.stat)
199 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
200 """Task to run arbitrary statistics.
202 The statistics should be measured by individual methods, and
203 add to the dictionary in the return struct.
207 inputExp : `lsst.afw.image.Exposure`
208 The exposure to measure.
209 ptc : `lsst.ip.isr.PtcDataset`, optional
210 A PTC object containing gains to use.
211 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
212 List of overscan results. Expected fields are:
215 Value or fit subtracted from the amplifier image data
216 (scalar or `lsst.afw.image.Image`).
218 Value or fit subtracted from the overscan image data
219 (scalar or `lsst.afw.image.Image`).
221 Image of the overscan region with the overscan
222 correction applied (`lsst.afw.image.Image`). This
223 quantity is used to estimate the amplifier read noise
228 resultStruct : `lsst.pipe.base.Struct`
229 Contains the measured statistics as a dict stored in a
230 field named ``results``.
235 Raised if the amplifier gains could not be found.
238 detector = inputExp.getDetector()
241 elif detector
is not None:
242 gains = {amp.getName(): amp.getGain()
for amp
in detector.getAmplifiers()}
244 raise RuntimeError(
"No source of gains provided.")
247 if self.config.doCtiStatistics:
248 ctiResults = self.
measureCti(inputExp, overscanResults, gains)
250 bandingResults =
None
251 if self.config.doBandingStatistics:
254 projectionResults =
None
255 if self.config.doProjectionStatistics:
258 calibDistributionResults =
None
259 if self.config.doCopyCalibDistributionStatistics:
262 biasShiftResults =
None
263 if self.config.doBiasShiftStatistics:
266 ampCorrelationResults =
None
267 if self.config.doAmplifierCorrelationStatistics:
270 mjd = inputExp.getMetadata().get(
"MJD",
None)
272 return pipeBase.Struct(
273 results={
"CTI": ctiResults,
274 "BANDING": bandingResults,
275 "PROJECTION": projectionResults,
276 "CALIBDIST": calibDistributionResults,
277 "BIASSHIFT": biasShiftResults,
278 "AMPCORR": ampCorrelationResults,
284 """Task to measure CTI statistics.
288 inputExp : `lsst.afw.image.Exposure`
290 overscans : `list` [`lsst.pipe.base.Struct`]
291 List of overscan results. Expected fields are:
294 Value or fit subtracted from the amplifier image data
295 (scalar or `lsst.afw.image.Image`).
297 Value or fit subtracted from the overscan image data
298 (scalar or `lsst.afw.image.Image`).
300 Image of the overscan region with the overscan
301 correction applied (`lsst.afw.image.Image`). This
302 quantity is used to estimate the amplifier read noise
304 gains : `dict` [`str` `float`]
305 Dictionary of per-amplifier gains, indexed by amplifier name.
309 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
310 Dictionary of measurements, keyed by amplifier name and
315 detector = inputExp.getDetector()
316 image = inputExp.image
319 assert len(overscans) == len(detector.getAmplifiers())
321 for ampIter, amp
in enumerate(detector.getAmplifiers()):
323 gain = gains[amp.getName()]
324 readoutCorner = amp.getReadoutCorner()
326 dataRegion = image[amp.getBBox()]
327 ampStats[
"IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.
statType,
331 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
334 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
340 if readoutCorner
in (ReadoutCorner.LR, ReadoutCorner.UR):
341 ampStats[
"FIRST_MEAN"] = pixelZ
342 ampStats[
"LAST_MEAN"] = pixelA
344 ampStats[
"FIRST_MEAN"] = pixelA
345 ampStats[
"LAST_MEAN"] = pixelZ
348 if overscans[ampIter]
is None:
351 self.log.warning(
"No overscan information available for ISR statistics for amp %s.",
353 nCols = amp.getRawSerialOverscanBBox().getWidth()
354 ampStats[
"OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
355 ampStats[
"OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
357 overscanImage = overscans[ampIter].overscanImage
360 for column
in range(0, overscanImage.getWidth()):
367 nRows = amp.getRawSerialOverscanBBox().getHeight()
368 osMean = afwMath.makeStatistics(overscanImage.image.array[:nRows, column],
370 columns.append(column)
371 if self.config.doApplyGainsForCtiStatistics:
372 values.append(gain * osMean)
374 values.append(osMean)
378 if readoutCorner
in (ReadoutCorner.LR, ReadoutCorner.UR):
379 ampStats[
"OVERSCAN_COLUMNS"] = list(reversed(columns))
380 ampStats[
"OVERSCAN_VALUES"] = list(reversed(values))
382 ampStats[
"OVERSCAN_COLUMNS"] = columns
383 ampStats[
"OVERSCAN_VALUES"] = values
385 outputStats[amp.getName()] = ampStats
410 """Task to measure banding statistics.
414 inputExp : `lsst.afw.image.Exposure`
416 overscans : `list` [`lsst.pipe.base.Struct`]
417 List of overscan results. Expected fields are:
420 Value or fit subtracted from the amplifier image data
421 (scalar or `lsst.afw.image.Image`).
423 Value or fit subtracted from the overscan image data
424 (scalar or `lsst.afw.image.Image`).
426 Image of the overscan region with the overscan
427 correction applied (`lsst.afw.image.Image`). This
428 quantity is used to estimate the amplifier read noise
433 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
434 Dictionary of measurements, keyed by amplifier name and
439 detector = inputExp.getDetector()
440 kernel = self.
makeKernel(self.config.bandingKernelSize)
442 outputStats[
"AMP_BANDING"] = []
443 for amp, overscanData
in zip(detector.getAmplifiers(), overscans):
444 overscanFit = np.array(overscanData.overscanFit)
445 overscanArray = overscanData.overscanImage.image.array
446 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
448 smoothedOverscan = np.convolve(rawOverscan, kernel, mode=
"valid")
450 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
451 self.config.bandingFractionHigh])
452 outputStats[
"AMP_BANDING"].append(float(high - low))
454 if self.config.bandingUseHalfDetector:
455 fullLength = len(outputStats[
"AMP_BANDING"])
456 outputStats[
"DET_BANDING"] = float(np.nanmedian(outputStats[
"AMP_BANDING"][0:fullLength//2]))
458 outputStats[
"DET_BANDING"] = float(np.nanmedian(outputStats[
"AMP_BANDING"]))
463 """Task to measure metrics from image slicing.
467 inputExp : `lsst.afw.image.Exposure`
469 overscans : `list` [`lsst.pipe.base.Struct`]
470 List of overscan results. Expected fields are:
473 Value or fit subtracted from the amplifier image data
474 (scalar or `lsst.afw.image.Image`).
476 Value or fit subtracted from the overscan image data
477 (scalar or `lsst.afw.image.Image`).
479 Image of the overscan region with the overscan
480 correction applied (`lsst.afw.image.Image`). This
481 quantity is used to estimate the amplifier read noise
486 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
487 Dictionary of measurements, keyed by amplifier name and
492 detector = inputExp.getDetector()
493 kernel = self.
makeKernel(self.config.projectionKernelSize)
495 outputStats[
"AMP_VPROJECTION"] = {}
496 outputStats[
"AMP_HPROJECTION"] = {}
497 convolveMode =
"valid"
498 if self.config.doProjectionFft:
499 outputStats[
"AMP_VFFT_REAL"] = {}
500 outputStats[
"AMP_VFFT_IMAG"] = {}
501 outputStats[
"AMP_HFFT_REAL"] = {}
502 outputStats[
"AMP_HFFT_IMAG"] = {}
503 convolveMode =
"same"
505 for amp
in detector.getAmplifiers():
506 ampArray = inputExp.image[amp.getBBox()].array
508 horizontalProjection = np.mean(ampArray, axis=0)
509 verticalProjection = np.mean(ampArray, axis=1)
511 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
512 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
514 outputStats[
"AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
515 outputStats[
"AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
517 if self.config.doProjectionFft:
518 horizontalWindow = np.ones_like(horizontalProjection)
519 verticalWindow = np.ones_like(verticalProjection)
520 if self.config.projectionFftWindow ==
"NONE":
522 elif self.config.projectionFftWindow ==
"HAMMING":
523 horizontalWindow = hamming(len(horizontalProjection))
524 verticalWindow = hamming(len(verticalProjection))
525 elif self.config.projectionFftWindow ==
"HANN":
526 horizontalWindow = hann(len(horizontalProjection))
527 verticalWindow = hann(len(verticalProjection))
528 elif self.config.projectionFftWindow ==
"GAUSSIAN":
529 horizontalWindow = gaussian(len(horizontalProjection))
530 verticalWindow = gaussian(len(verticalProjection))
532 raise RuntimeError(f
"Invalid window function: {self.config.projectionFftWindow}")
534 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
535 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
537 outputStats[
"AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
538 outputStats[
"AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
539 outputStats[
"AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
540 outputStats[
"AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
545 """Copy calibration statistics for this exposure.
549 inputExp : `lsst.afw.image.Exposure`
550 The exposure being processed.
552 Keyword arguments with calibrations.
556 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
557 Dictionary of measurements, keyed by amplifier name and
562 for amp
in inputExp.getDetector():
565 for calibType
in (
"bias",
"dark",
"flat"):
566 if kwargs.get(calibType,
None)
is not None:
567 metadata = kwargs[calibType].getMetadata()
568 for pct
in self.config.expectedDistributionLevels:
569 key = f
"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
570 ampStats[key] = metadata.get(key, np.nan)
571 outputStats[amp.getName()] = ampStats
575 """Measure number of bias shifts from overscan data.
579 inputExp : `lsst.afw.image.Exposure`
581 overscans : `list` [`lsst.pipe.base.Struct`]
582 List of overscan results. Expected fields are:
585 Value or fit subtracted from the amplifier image data
586 (scalar or `lsst.afw.image.Image`).
588 Value or fit subtracted from the overscan image data
589 (scalar or `lsst.afw.image.Image`).
591 Image of the overscan region with the overscan
592 correction applied (`lsst.afw.image.Image`). This
593 quantity is used to estimate the amplifier read noise
598 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
599 Dictionary of measurements, keyed by amplifier name and
604 Based on eop_pipe implementation:
605 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505
609 detector = inputExp.getDetector()
610 for amp, overscans
in zip(detector, overscanResults):
613 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit
616 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1)
620 ampStats[
"LOCAL_NOISE"] = float(noise)
621 ampStats[
"BIAS_SHIFTS"] = shift_peaks
623 outputStats[amp.getName()] = ampStats
627 """Scan overscan data for shifts.
631 overscanData : `list` [`float`]
632 Overscan data to search for shifts.
637 Noise estimated from Butterworth filtered overscan data.
638 peaks : `list` [`float`, `float`, `int`, `int`]
639 Shift peak information, containing the convolved peak
640 value, the raw peak value, and the lower and upper bounds
641 of the region checked.
643 numerator, denominator = butter(self.config.biasShiftFilterOrder,
644 self.config.biasShiftCutoff,
645 btype=
"high", analog=
False)
646 noise = np.std(filtfilt(numerator, denominator, overscanData))
647 kernel = np.concatenate([np.arange(self.config.biasShiftWindow),
648 np.arange(-self.config.biasShiftWindow + 1, 0)])
649 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow])
651 convolved = np.convolve(overscanData, kernel, mode=
"valid")
652 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow))
654 shift_check = np.abs(convolved)/noise
655 shift_mask = shift_check > self.config.biasShiftThreshold
656 shift_mask[:self.config.biasShiftRowSkip] =
False
658 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0),
659 shift_mask.view(np.int8),
660 np.int8(0)])).reshape(-1, 2)
662 for region
in shift_regions:
663 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0]
666 [float(convolved[region_peak]), float(region_peak),
667 int(region[0]), int(region[1])])
668 return noise, shift_peaks
671 """Determine if a region is flat.
676 Row with possible peak.
678 Value at the possible peak.
679 overscanData : `list` [`float`]
680 Overscan data used to fit around the possible peak.
685 Indicates if the region is flat, and so the peak is valid.
687 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow)
688 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow)
690 preFit = linregress(prerange, overscanData[prerange])
691 postFit = linregress(postrange, overscanData[postrange])
694 preTrend = (2*preFit[0]*len(prerange) < shiftPeak)
695 postTrend = (2*postFit[0]*len(postrange) < shiftPeak)
697 preTrend = (2*preFit[0]*len(prerange) > shiftPeak)
698 postTrend = (2*postFit[0]*len(postrange) > shiftPeak)
700 return (preTrend
and postTrend)
703 """Measure correlations between amplifier segments.
707 inputExp : `lsst.afw.image.Exposure`
709 overscans : `list` [`lsst.pipe.base.Struct`]
710 List of overscan results. Expected fields are:
713 Value or fit subtracted from the amplifier image data
714 (scalar or `lsst.afw.image.Image`).
716 Value or fit subtracted from the overscan image data
717 (scalar or `lsst.afw.image.Image`).
719 Image of the overscan region with the overscan
720 correction applied (`lsst.afw.image.Image`). This
721 quantity is used to estimate the amplifier read noise
726 outputStats : `dict` [`str`, [`dict` [`str`,`float`]]
727 Dictionary of measurements, keyed by amplifier name and
732 Based on eo_pipe implementation:
733 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505
737 detector = inputExp.getDetector()
739 serialOSCorr = np.empty((len(detector), len(detector)))
740 imageCorr = np.empty((len(detector), len(detector)))
741 for ampId, overscan
in enumerate(overscanResults):
742 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit
743 rawOverscan = rawOverscan.ravel()
745 ampImage = inputExp[detector[ampId].getBBox()]
746 ampImage = ampImage.image.array.ravel()
748 for ampId2, overscan2
in enumerate(overscanResults):
751 serialOSCorr[ampId, ampId2] = 1.0
752 imageCorr[ampId, ampId2] = 1.0
754 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit
755 rawOverscan2 = rawOverscan2.ravel()
757 serialOSCorr[ampId, ampId2] = np.corrcoef(rawOverscan, rawOverscan2)[0, 1]
759 ampImage2 = inputExp[detector[ampId2].getBBox()]
760 ampImage2 = ampImage2.image.array.ravel()
762 imageCorr[ampId, ampId2] = np.corrcoef(ampImage, ampImage2)[0, 1]
764 outputStats[
"OVERSCAN_CORR"] = serialOSCorr.tolist()
765 outputStats[
"IMAGE_CORR"] = imageCorr.tolist()