195 self.
statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter,
196 afwImage.Mask.getPlaneBitMask(self.config.badMask))
197 self.
statType = afwMath.stringToStatisticsProperty(self.config.stat)
199 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
200 """Task to run arbitrary statistics.
202 The statistics should be measured by individual methods, and
203 add to the dictionary in the return struct.
207 inputExp : `lsst.afw.image.Exposure`
208 The exposure to measure.
209 ptc : `lsst.ip.isr.PtcDataset`, optional
210 A PTC object containing gains to use.
211 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
212 List of overscan results. Expected fields are:
215 Value or fit subtracted from the amplifier image data
216 (scalar or `lsst.afw.image.Image`).
218 Value or fit subtracted from the overscan image data
219 (scalar or `lsst.afw.image.Image`).
221 Image of the overscan region with the overscan
222 correction applied (`lsst.afw.image.Image`). This
223 quantity is used to estimate the amplifier read noise
228 resultStruct : `lsst.pipe.base.Struct`
229 Contains the measured statistics as a dict stored in a
230 field named ``results``.
235 Raised if the amplifier gains could not be found.
238 detector = inputExp.getDetector()
241 elif detector
is not None:
242 gains = {amp.getName(): amp.getGain()
for amp
in detector.getAmplifiers()}
244 raise RuntimeError(
"No source of gains provided.")
247 if self.config.doCtiStatistics:
248 ctiResults = self.
measureCti(inputExp, overscanResults, gains)
250 bandingResults =
None
251 if self.config.doBandingStatistics:
254 projectionResults =
None
255 if self.config.doProjectionStatistics:
258 calibDistributionResults =
None
259 if self.config.doCopyCalibDistributionStatistics:
262 biasShiftResults =
None
263 if self.config.doBiasShiftStatistics:
266 ampCorrelationResults =
None
267 if self.config.doAmplifierCorrelationStatistics:
270 return pipeBase.Struct(
271 results={
"CTI": ctiResults,
272 "BANDING": bandingResults,
273 "PROJECTION": projectionResults,
274 "CALIBDIST": calibDistributionResults,
275 "BIASSHIFT": biasShiftResults,
276 "AMPCORR": ampCorrelationResults,
281 """Task to measure CTI statistics.
285 inputExp : `lsst.afw.image.Exposure`
287 overscans : `list` [`lsst.pipe.base.Struct`]
288 List of overscan results. Expected fields are:
291 Value or fit subtracted from the amplifier image data
292 (scalar or `lsst.afw.image.Image`).
294 Value or fit subtracted from the overscan image data
295 (scalar or `lsst.afw.image.Image`).
297 Image of the overscan region with the overscan
298 correction applied (`lsst.afw.image.Image`). This
299 quantity is used to estimate the amplifier read noise
301 gains : `dict` [`str` `float`]
302 Dictionary of per-amplifier gains, indexed by amplifier name.
306 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
307 Dictionary of measurements, keyed by amplifier name and
312 detector = inputExp.getDetector()
313 image = inputExp.image
316 assert len(overscans) == len(detector.getAmplifiers())
318 for ampIter, amp
in enumerate(detector.getAmplifiers()):
320 gain = gains[amp.getName()]
321 readoutCorner = amp.getReadoutCorner()
323 dataRegion = image[amp.getBBox()]
324 ampStats[
"IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.
statType,
328 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
331 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
337 if readoutCorner
in (ReadoutCorner.LR, ReadoutCorner.UR):
338 ampStats[
"FIRST_MEAN"] = pixelZ
339 ampStats[
"LAST_MEAN"] = pixelA
341 ampStats[
"FIRST_MEAN"] = pixelA
342 ampStats[
"LAST_MEAN"] = pixelZ
345 if overscans[ampIter]
is None:
348 self.log.warning(
"No overscan information available for ISR statistics for amp %s.",
350 nCols = amp.getSerialOverscanBBox().getWidth()
351 ampStats[
"OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
352 ampStats[
"OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
354 overscanImage = overscans[ampIter].overscanImage
357 for column
in range(0, overscanImage.getWidth()):
358 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column],
360 columns.append(column)
361 if self.config.doApplyGainsForCtiStatistics:
362 values.append(gain * osMean)
364 values.append(osMean)
368 if readoutCorner
in (ReadoutCorner.LR, ReadoutCorner.UR):
369 ampStats[
"OVERSCAN_COLUMNS"] = list(reversed(columns))
370 ampStats[
"OVERSCAN_VALUES"] = list(reversed(values))
372 ampStats[
"OVERSCAN_COLUMNS"] = columns
373 ampStats[
"OVERSCAN_VALUES"] = values
375 outputStats[amp.getName()] = ampStats
400 """Task to measure banding statistics.
404 inputExp : `lsst.afw.image.Exposure`
406 overscans : `list` [`lsst.pipe.base.Struct`]
407 List of overscan results. Expected fields are:
410 Value or fit subtracted from the amplifier image data
411 (scalar or `lsst.afw.image.Image`).
413 Value or fit subtracted from the overscan image data
414 (scalar or `lsst.afw.image.Image`).
416 Image of the overscan region with the overscan
417 correction applied (`lsst.afw.image.Image`). This
418 quantity is used to estimate the amplifier read noise
423 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
424 Dictionary of measurements, keyed by amplifier name and
429 detector = inputExp.getDetector()
430 kernel = self.
makeKernel(self.config.bandingKernelSize)
432 outputStats[
"AMP_BANDING"] = []
433 for amp, overscanData
in zip(detector.getAmplifiers(), overscans):
434 overscanFit = np.array(overscanData.overscanFit)
435 overscanArray = overscanData.overscanImage.image.array
436 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
438 smoothedOverscan = np.convolve(rawOverscan, kernel, mode=
"valid")
440 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
441 self.config.bandingFractionHigh])
442 outputStats[
"AMP_BANDING"].append(float(high - low))
444 if self.config.bandingUseHalfDetector:
445 fullLength = len(outputStats[
"AMP_BANDING"])
446 outputStats[
"DET_BANDING"] = float(np.nanmedian(outputStats[
"AMP_BANDING"][0:fullLength//2]))
448 outputStats[
"DET_BANDING"] = float(np.nanmedian(outputStats[
"AMP_BANDING"]))
453 """Task to measure metrics from image slicing.
457 inputExp : `lsst.afw.image.Exposure`
459 overscans : `list` [`lsst.pipe.base.Struct`]
460 List of overscan results. Expected fields are:
463 Value or fit subtracted from the amplifier image data
464 (scalar or `lsst.afw.image.Image`).
466 Value or fit subtracted from the overscan image data
467 (scalar or `lsst.afw.image.Image`).
469 Image of the overscan region with the overscan
470 correction applied (`lsst.afw.image.Image`). This
471 quantity is used to estimate the amplifier read noise
476 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
477 Dictionary of measurements, keyed by amplifier name and
482 detector = inputExp.getDetector()
483 kernel = self.
makeKernel(self.config.projectionKernelSize)
485 outputStats[
"AMP_VPROJECTION"] = {}
486 outputStats[
"AMP_HPROJECTION"] = {}
487 convolveMode =
"valid"
488 if self.config.doProjectionFft:
489 outputStats[
"AMP_VFFT_REAL"] = {}
490 outputStats[
"AMP_VFFT_IMAG"] = {}
491 outputStats[
"AMP_HFFT_REAL"] = {}
492 outputStats[
"AMP_HFFT_IMAG"] = {}
493 convolveMode =
"same"
495 for amp
in detector.getAmplifiers():
496 ampArray = inputExp.image[amp.getBBox()].array
498 horizontalProjection = np.mean(ampArray, axis=0)
499 verticalProjection = np.mean(ampArray, axis=1)
501 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
502 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
504 outputStats[
"AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
505 outputStats[
"AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
507 if self.config.doProjectionFft:
508 horizontalWindow = np.ones_like(horizontalProjection)
509 verticalWindow = np.ones_like(verticalProjection)
510 if self.config.projectionFftWindow ==
"NONE":
512 elif self.config.projectionFftWindow ==
"HAMMING":
513 horizontalWindow = hamming(len(horizontalProjection))
514 verticalWindow = hamming(len(verticalProjection))
515 elif self.config.projectionFftWindow ==
"HANN":
516 horizontalWindow = hann(len(horizontalProjection))
517 verticalWindow = hann(len(verticalProjection))
518 elif self.config.projectionFftWindow ==
"GAUSSIAN":
519 horizontalWindow = gaussian(len(horizontalProjection))
520 verticalWindow = gaussian(len(verticalProjection))
522 raise RuntimeError(f
"Invalid window function: {self.config.projectionFftWindow}")
524 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
525 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
527 outputStats[
"AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
528 outputStats[
"AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
529 outputStats[
"AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
530 outputStats[
"AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
535 """Copy calibration statistics for this exposure.
539 inputExp : `lsst.afw.image.Exposure`
540 The exposure being processed.
542 Keyword arguments with calibrations.
546 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
547 Dictionary of measurements, keyed by amplifier name and
552 for amp
in inputExp.getDetector():
555 for calibType
in (
"bias",
"dark",
"flat"):
556 if kwargs.get(calibType,
None)
is not None:
557 metadata = kwargs[calibType].getMetadata()
558 for pct
in self.config.expectedDistributionLevels:
559 key = f
"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
560 ampStats[key] = metadata.get(key, np.nan)
561 outputStats[amp.getName()] = ampStats
565 """Measure number of bias shifts from overscan data.
569 inputExp : `lsst.afw.image.Exposure`
571 overscans : `list` [`lsst.pipe.base.Struct`]
572 List of overscan results. Expected fields are:
575 Value or fit subtracted from the amplifier image data
576 (scalar or `lsst.afw.image.Image`).
578 Value or fit subtracted from the overscan image data
579 (scalar or `lsst.afw.image.Image`).
581 Image of the overscan region with the overscan
582 correction applied (`lsst.afw.image.Image`). This
583 quantity is used to estimate the amplifier read noise
588 outputStats : `dict` [`str`, [`dict` [`str`,`float]]
589 Dictionary of measurements, keyed by amplifier name and
594 Based on eop_pipe implementation:
595 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505
599 detector = inputExp.getDetector()
600 for amp, overscans
in zip(detector, overscanResults):
603 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit
606 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1)
610 ampStats[
"LOCAL_NOISE"] = float(noise)
611 ampStats[
"BIAS_SHIFTS"] = shift_peaks
613 outputStats[amp.getName()] = ampStats
617 """Scan overscan data for shifts.
621 overscanData : `list` [`float`]
622 Overscan data to search for shifts.
627 Noise estimated from Butterworth filtered overscan data.
628 peaks : `list` [`float`, `float`, `int`, `int`]
629 Shift peak information, containing the convolved peak
630 value, the raw peak value, and the lower and upper bounds
631 of the region checked.
633 numerator, denominator = butter(self.config.biasShiftFilterOrder,
634 self.config.biasShiftCutoff,
635 btype=
"high", analog=
False)
636 noise = np.std(filtfilt(numerator, denominator, overscanData))
637 kernel = np.concatenate([np.arange(self.config.biasShiftWindow),
638 np.arange(-self.config.biasShiftWindow + 1, 0)])
639 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow])
641 convolved = np.convolve(overscanData, kernel, mode=
"valid")
642 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow))
644 shift_check = np.abs(convolved)/noise
645 shift_mask = shift_check > self.config.biasShiftThreshold
646 shift_mask[:self.config.biasShiftRowSkip] =
False
648 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0),
649 shift_mask.view(np.int8),
650 np.int8(0)])).reshape(-1, 2)
652 for region
in shift_regions:
653 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0]
656 [float(convolved[region_peak]), float(region_peak),
657 int(region[0]), int(region[1])])
658 return noise, shift_peaks
661 """Determine if a region is flat.
666 Row with possible peak.
668 Value at the possible peak.
669 overscanData : `list` [`float`]
670 Overscan data used to fit around the possible peak.
675 Indicates if the region is flat, and so the peak is valid.
677 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow)
678 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow)
680 preFit = linregress(prerange, overscanData[prerange])
681 postFit = linregress(postrange, overscanData[postrange])
684 preTrend = (2*preFit[0]*len(prerange) < shiftPeak)
685 postTrend = (2*postFit[0]*len(postrange) < shiftPeak)
687 preTrend = (2*preFit[0]*len(prerange) > shiftPeak)
688 postTrend = (2*postFit[0]*len(postrange) > shiftPeak)
690 return (preTrend
and postTrend)
693 """Measure correlations between amplifier segments.
697 inputExp : `lsst.afw.image.Exposure`
699 overscans : `list` [`lsst.pipe.base.Struct`]
700 List of overscan results. Expected fields are:
703 Value or fit subtracted from the amplifier image data
704 (scalar or `lsst.afw.image.Image`).
706 Value or fit subtracted from the overscan image data
707 (scalar or `lsst.afw.image.Image`).
709 Image of the overscan region with the overscan
710 correction applied (`lsst.afw.image.Image`). This
711 quantity is used to estimate the amplifier read noise
716 outputStats : `dict` [`str`, [`dict` [`str`,`float`]]
717 Dictionary of measurements, keyed by amplifier name and
722 Based on eo_pipe implementation:
723 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505
727 detector = inputExp.getDetector()
729 serialOSCorr = np.empty((len(detector), len(detector)))
730 imageCorr = np.empty((len(detector), len(detector)))
731 for ampId, overscan
in enumerate(overscanResults):
732 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit
733 rawOverscan = rawOverscan.ravel()
735 ampImage = inputExp[detector[ampId].getBBox()]
736 ampImage = ampImage.image.array.ravel()
738 for ampId2, overscan2
in enumerate(overscanResults):
741 serialOSCorr[ampId, ampId2] = 1.0
742 imageCorr[ampId, ampId2] = 1.0
744 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit
745 rawOverscan2 = rawOverscan2.ravel()
747 serialOSCorr[ampId, ampId2] = np.corrcoef(rawOverscan, rawOverscan2)[0, 1]
749 ampImage2 = inputExp[detector[ampId2].getBBox()]
750 ampImage2 = ampImage2.image.array.ravel()
752 imageCorr[ampId, ampId2] = np.corrcoef(ampImage, ampImage2)[0, 1]
754 outputStats[
"OVERSCAN_CORR"] = serialOSCorr.tolist()
755 outputStats[
"IMAGE_CORR"] = imageCorr.tolist()