223 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs):
224 """Task to run arbitrary statistics.
226 The statistics should be measured by individual methods, and
227 add to the dictionary in the return struct.
231 inputExp : `lsst.afw.image.Exposure`
232 The exposure to measure.
233 ptc : `lsst.ip.isr.PtcDataset`, optional
234 A PTC object containing gains to use.
235 overscanResults : `list` [`lsst.pipe.base.Struct`], optional
236 List of overscan results. Expected fields are:
239 Value or fit subtracted from the amplifier image data
240 (scalar or `lsst.afw.image.Image`).
242 Value or fit subtracted from the overscan image data
243 (scalar or `lsst.afw.image.Image`).
245 Image of the overscan region with the overscan
246 correction applied (`lsst.afw.image.Image`). This
247 quantity is used to estimate the amplifier read noise
250 Keyword arguments. Calibrations being passed in should
255 resultStruct : `lsst.pipe.base.Struct`
256 Contains the measured statistics as a dict stored in a
257 field named ``results``.
262 Raised if the amplifier gains could not be found.
265 detector = inputExp.getDetector()
268 elif detector
is not None:
269 gains = {amp.getName(): amp.getGain()
for amp
in detector.getAmplifiers()}
271 raise RuntimeError(
"No source of gains provided.")
274 if self.config.doCtiStatistics:
275 ctiResults = self.
measureCti(inputExp, overscanResults, gains)
277 bandingResults =
None
278 if self.config.doBandingStatistics:
281 projectionResults =
None
282 if self.config.doProjectionStatistics:
285 divisaderoResults =
None
286 if self.config.doDivisaderoStatistics:
289 calibDistributionResults =
None
290 if self.config.doCopyCalibDistributionStatistics:
293 biasShiftResults =
None
294 if self.config.doBiasShiftStatistics:
297 ampCorrelationResults =
None
298 if self.config.doAmplifierCorrelationStatistics:
301 mjd = inputExp.getMetadata().get(
"MJD",
None)
303 return pipeBase.Struct(
304 results={
"CTI": ctiResults,
305 "BANDING": bandingResults,
306 "PROJECTION": projectionResults,
307 "CALIBDIST": calibDistributionResults,
308 "BIASSHIFT": biasShiftResults,
309 "AMPCORR": ampCorrelationResults,
311 'DIVISADERO': divisaderoResults,
316 """Task to measure CTI statistics.
320 inputExp : `lsst.afw.image.Exposure`
322 overscans : `list` [`lsst.pipe.base.Struct`]
323 List of overscan results. Expected fields are:
326 Value or fit subtracted from the amplifier image data
327 (scalar or `lsst.afw.image.Image`).
329 Value or fit subtracted from the overscan image data
330 (scalar or `lsst.afw.image.Image`).
332 Image of the overscan region with the overscan
333 correction applied (`lsst.afw.image.Image`). This
334 quantity is used to estimate the amplifier read noise
336 gains : `dict` [`str` `float`]
337 Dictionary of per-amplifier gains, indexed by amplifier name.
341 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
342 Dictionary of measurements, keyed by amplifier name and
347 detector = inputExp.getDetector()
348 image = inputExp.image
351 assert len(overscans) == len(detector.getAmplifiers())
353 for ampIter, amp
in enumerate(detector.getAmplifiers()):
355 gain = gains[amp.getName()]
356 readoutCorner = amp.getReadoutCorner()
358 dataRegion = image[amp.getBBox()]
359 ampStats[
"IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.
statType,
363 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0],
366 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1],
372 if readoutCorner
in (ReadoutCorner.LR, ReadoutCorner.UR):
373 ampStats[
"FIRST_MEAN"] = pixelZ
374 ampStats[
"LAST_MEAN"] = pixelA
376 ampStats[
"FIRST_MEAN"] = pixelA
377 ampStats[
"LAST_MEAN"] = pixelZ
380 if overscans[ampIter]
is None:
383 self.log.warning(
"No overscan information available for ISR statistics for amp %s.",
385 nCols = amp.getRawSerialOverscanBBox().getWidth()
386 ampStats[
"OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan)
387 ampStats[
"OVERSCAN_VALUES"] = np.full((nCols, ), np.nan)
389 overscanImage = overscans[ampIter].overscanImage
392 for column
in range(0, overscanImage.getWidth()):
399 nRows = amp.getRawSerialOverscanBBox().getHeight()
400 osMean = afwMath.makeStatistics(overscanImage.image.array[:nRows, column],
402 columns.append(column)
403 if self.config.doApplyGainsForCtiStatistics:
404 values.append(gain * osMean)
406 values.append(osMean)
410 if readoutCorner
in (ReadoutCorner.LR, ReadoutCorner.UR):
411 ampStats[
"OVERSCAN_COLUMNS"] = list(reversed(columns))
412 ampStats[
"OVERSCAN_VALUES"] = list(reversed(values))
414 ampStats[
"OVERSCAN_COLUMNS"] = columns
415 ampStats[
"OVERSCAN_VALUES"] = values
417 outputStats[amp.getName()] = ampStats
442 """Task to measure banding statistics.
446 inputExp : `lsst.afw.image.Exposure`
448 overscans : `list` [`lsst.pipe.base.Struct`]
449 List of overscan results. Expected fields are:
452 Value or fit subtracted from the amplifier image data
453 (scalar or `lsst.afw.image.Image`).
455 Value or fit subtracted from the overscan image data
456 (scalar or `lsst.afw.image.Image`).
458 Image of the overscan region with the overscan
459 correction applied (`lsst.afw.image.Image`). This
460 quantity is used to estimate the amplifier read noise
465 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
466 Dictionary of measurements, keyed by amplifier name and
471 detector = inputExp.getDetector()
472 kernel = self.
makeKernel(self.config.bandingKernelSize)
474 outputStats[
"AMP_BANDING"] = []
475 for amp, overscanData
in zip(detector.getAmplifiers(), overscans):
476 overscanFit = np.array(overscanData.overscanFit)
477 overscanArray = overscanData.overscanImage.image.array
478 rawOverscan = np.mean(overscanArray + overscanFit, axis=1)
480 smoothedOverscan = np.convolve(rawOverscan, kernel, mode=
"valid")
482 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow,
483 self.config.bandingFractionHigh])
484 outputStats[
"AMP_BANDING"].append(float(high - low))
486 if self.config.bandingUseHalfDetector:
487 fullLength = len(outputStats[
"AMP_BANDING"])
488 outputStats[
"DET_BANDING"] = float(np.nanmedian(outputStats[
"AMP_BANDING"][0:fullLength//2]))
490 outputStats[
"DET_BANDING"] = float(np.nanmedian(outputStats[
"AMP_BANDING"]))
495 """Task to measure metrics from image slicing.
499 inputExp : `lsst.afw.image.Exposure`
501 overscans : `list` [`lsst.pipe.base.Struct`]
502 List of overscan results. Expected fields are:
505 Value or fit subtracted from the amplifier image data
506 (scalar or `lsst.afw.image.Image`).
508 Value or fit subtracted from the overscan image data
509 (scalar or `lsst.afw.image.Image`).
511 Image of the overscan region with the overscan
512 correction applied (`lsst.afw.image.Image`). This
513 quantity is used to estimate the amplifier read noise
518 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
519 Dictionary of measurements, keyed by amplifier name and
524 detector = inputExp.getDetector()
525 kernel = self.
makeKernel(self.config.projectionKernelSize)
527 outputStats[
"AMP_VPROJECTION"] = {}
528 outputStats[
"AMP_HPROJECTION"] = {}
529 convolveMode =
"valid"
530 if self.config.doProjectionFft:
531 outputStats[
"AMP_VFFT_REAL"] = {}
532 outputStats[
"AMP_VFFT_IMAG"] = {}
533 outputStats[
"AMP_HFFT_REAL"] = {}
534 outputStats[
"AMP_HFFT_IMAG"] = {}
535 convolveMode =
"same"
537 for amp
in detector.getAmplifiers():
538 ampArray = inputExp.image[amp.getBBox()].array
540 horizontalProjection = np.mean(ampArray, axis=0)
541 verticalProjection = np.mean(ampArray, axis=1)
543 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode)
544 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode)
546 outputStats[
"AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist()
547 outputStats[
"AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist()
549 if self.config.doProjectionFft:
550 horizontalWindow = np.ones_like(horizontalProjection)
551 verticalWindow = np.ones_like(verticalProjection)
552 if self.config.projectionFftWindow ==
"NONE":
554 elif self.config.projectionFftWindow ==
"HAMMING":
555 horizontalWindow = hamming(len(horizontalProjection))
556 verticalWindow = hamming(len(verticalProjection))
557 elif self.config.projectionFftWindow ==
"HANN":
558 horizontalWindow = hann(len(horizontalProjection))
559 verticalWindow = hann(len(verticalProjection))
560 elif self.config.projectionFftWindow ==
"GAUSSIAN":
561 horizontalWindow = gaussian(len(horizontalProjection))
562 verticalWindow = gaussian(len(verticalProjection))
564 raise RuntimeError(f
"Invalid window function: {self.config.projectionFftWindow}")
566 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow))
567 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow))
569 outputStats[
"AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist()
570 outputStats[
"AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist()
571 outputStats[
"AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist()
572 outputStats[
"AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist()
577 """Copy calibration statistics for this exposure.
581 inputExp : `lsst.afw.image.Exposure`
582 The exposure being processed.
584 Keyword arguments with calibrations.
588 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
589 Dictionary of measurements, keyed by amplifier name and
595 for amp
in inputExp.getDetector():
598 for calibType
in (
"bias",
"dark",
"flat"):
599 if kwargs.get(calibType,
None)
is not None:
600 metadata = kwargs[calibType].getMetadata()
601 for pct
in self.config.expectedDistributionLevels:
602 key = f
"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT"
603 ampStats[key] = metadata.get(key, np.nan)
605 for calibType
in (
"defects"):
606 if kwargs.get(calibType,
None)
is not None:
607 metadata = kwargs[calibType].getMetadata()
608 for key
in (f
"LSST CALIB {calibType.upper()} {amp.getName()} N_HOT",
609 f
"LSST CALIB {calibType.upper()} {amp.getName()} N_COLD"):
610 ampStats[key] = metadata.get(key, np.nan)
611 outputStats[amp.getName()] = ampStats
614 for calibType
in (
"defects"):
615 if kwargs.get(calibType,
None)
is not None:
616 metadata = kwargs[calibType].getMetadata()
617 for key
in (f
"LSST CALIB {calibType.upper()} N_BAD_COLUMNS"):
618 outputStats[
"detector"][key] = metadata.get(key, np.nan)
623 """Measure number of bias shifts from overscan data.
627 inputExp : `lsst.afw.image.Exposure`
629 overscans : `list` [`lsst.pipe.base.Struct`]
630 List of overscan results. Expected fields are:
633 Value or fit subtracted from the amplifier image data
634 (scalar or `lsst.afw.image.Image`).
636 Value or fit subtracted from the overscan image data
637 (scalar or `lsst.afw.image.Image`).
639 Image of the overscan region with the overscan
640 correction applied (`lsst.afw.image.Image`). This
641 quantity is used to estimate the amplifier read noise
646 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
647 Dictionary of measurements, keyed by amplifier name and
652 Based on eop_pipe implementation:
653 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505
657 detector = inputExp.getDetector()
658 for amp, overscans
in zip(detector, overscanResults):
661 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit
664 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1)
668 ampStats[
"LOCAL_NOISE"] = float(noise)
669 ampStats[
"BIAS_SHIFTS"] = shift_peaks
671 outputStats[amp.getName()] = ampStats
675 """Scan overscan data for shifts.
679 overscanData : `list` [`float`]
680 Overscan data to search for shifts.
685 Noise estimated from Butterworth filtered overscan data.
686 peaks : `list` [`float`, `float`, `int`, `int`]
687 Shift peak information, containing the convolved peak
688 value, the raw peak value, and the lower and upper bounds
689 of the region checked.
691 numerator, denominator = butter(self.config.biasShiftFilterOrder,
692 self.config.biasShiftCutoff,
693 btype=
"high", analog=
False)
694 noise = np.std(filtfilt(numerator, denominator, overscanData))
695 kernel = np.concatenate([np.arange(self.config.biasShiftWindow),
696 np.arange(-self.config.biasShiftWindow + 1, 0)])
697 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow])
699 convolved = np.convolve(overscanData, kernel, mode=
"valid")
700 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow))
702 shift_check = np.abs(convolved)/noise
703 shift_mask = shift_check > self.config.biasShiftThreshold
704 shift_mask[:self.config.biasShiftRowSkip] =
False
706 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0),
707 shift_mask.view(np.int8),
708 np.int8(0)])).reshape(-1, 2)
710 for region
in shift_regions:
711 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0]
714 [float(convolved[region_peak]), float(region_peak),
715 int(region[0]), int(region[1])])
716 return noise, shift_peaks
719 """Determine if a region is flat.
724 Row with possible peak.
726 Value at the possible peak.
727 overscanData : `list` [`float`]
728 Overscan data used to fit around the possible peak.
733 Indicates if the region is flat, and so the peak is valid.
735 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow)
736 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow)
738 preFit = linregress(prerange, overscanData[prerange])
739 postFit = linregress(postrange, overscanData[postrange])
742 preTrend = (2*preFit[0]*len(prerange) < shiftPeak)
743 postTrend = (2*postFit[0]*len(postrange) < shiftPeak)
745 preTrend = (2*preFit[0]*len(prerange) > shiftPeak)
746 postTrend = (2*postFit[0]*len(postrange) > shiftPeak)
748 return (preTrend
and postTrend)
751 """Measure correlations between amplifier segments.
755 inputExp : `lsst.afw.image.Exposure`
757 overscans : `list` [`lsst.pipe.base.Struct`]
758 List of overscan results. Expected fields are:
761 Value or fit subtracted from the amplifier image data
762 (scalar or `lsst.afw.image.Image`).
764 Value or fit subtracted from the overscan image data
765 (scalar or `lsst.afw.image.Image`).
767 Image of the overscan region with the overscan
768 correction applied (`lsst.afw.image.Image`). This
769 quantity is used to estimate the amplifier read noise
774 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
775 Dictionary of measurements, keyed by amplifier name and
780 Based on eo_pipe implementation:
781 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505
783 detector = inputExp.getDetector()
786 for ampId, overscan
in enumerate(overscanResults):
787 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit
788 rawOverscan = rawOverscan.ravel()
790 ampImage = inputExp[detector[ampId].getBBox()]
791 ampImage = ampImage.image.array.ravel()
793 for ampId2, overscan2
in enumerate(overscanResults):
797 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit
798 rawOverscan2 = rawOverscan2.ravel()
800 osC = np.corrcoef(rawOverscan, rawOverscan2)[0, 1]
802 ampImage2 = inputExp[detector[ampId2].getBBox()]
803 ampImage2 = ampImage2.image.array.ravel()
805 imC = np.corrcoef(ampImage, ampImage2)[0, 1]
807 {
'detector': detector.getId(),
808 'detectorComp': detector.getId(),
809 'ampName': detector[ampId].getName(),
810 'ampComp': detector[ampId2].getName(),
811 'imageCorr': float(imC),
812 'overscanCorr': float(osC),
818 """Measure Max Divisadero Tearing effect per amp.
822 inputExp : `lsst.afw.image.Exposure`
823 Exposure to measure. Usually a flat.
825 The flat will be selected from here.
829 outputStats : `dict` [`str`, [`dict` [`str`, `float`]]]
830 Dictionary of measurements, keyed by amplifier name and
834 - DIVISADERO_PROFILE: Robust mean of rows between
835 divisaderoProjection<Maximum|Minumum> on readout edge of ccd
836 normalized by a linear fit to the same rows.
837 - DIVISADERO_MAX_PAIR: Tuple of maximum of the absolute values of
838 the DIVISADERO_PROFILE, for number of pixels (specified by
839 divisaderoNumImpactPixels on left and right side of amp.
840 - DIVISADERO_MAX: Maximum of the absolute values of the
841 the DIVISADERO_PROFILE, for the divisaderoNumImpactPixels on
842 boundaries of neighboring amps (including the pixels in those
843 neighborboring amps).
847 for amp
in inputExp.getDetector():
849 ampArray = inputExp.image[amp.getBBox()].array
851 if amp.getReadoutCorner().name
in (
'UL',
'UR'):
852 minRow = amp.getBBox().getHeight() - self.config.divisaderoProjectionMaximum
853 maxRow = amp.getBBox().getHeight() - self.config.divisaderoProjectionMinimum
855 minRow = self.config.divisaderoProjectionMinimum
856 maxRow = self.config.divisaderoProjectionMaximum
858 segment = slice(minRow, maxRow)
859 projection, _, _ = astropy.stats.sigma_clipped_stats(ampArray[segment, :], axis=0)
862 projection /= np.median(projection)
863 columns = np.arange(len(projection))
865 segment = slice(self.config.divisaderoEdgePixels, -self.config.divisaderoEdgePixels)
866 model = np.polyfit(columns[segment], projection[segment], 1)
867 modelProjection = model[0] * columns + model[1]
868 divisaderoProfile = projection / modelProjection
871 leftMax = np.nanmax(np.abs(divisaderoProfile[0:self.config.divisaderoNumImpactPixels] - 1.0))
872 rightMax = np.nanmax(np.abs(divisaderoProfile[-self.config.divisaderoNumImpactPixels:] - 1.0))
874 ampStats[
'DIVISADERO_PROFILE'] = np.array(divisaderoProfile).tolist()
875 ampStats[
'DIVISADERO_MAX_PAIR'] = [leftMax, rightMax]
876 outputStats[amp.getName()] = ampStats
878 detector = inputExp.getDetector()
879 xCenters = [amp.getBBox().getCenterX()
for amp
in detector]
880 yCenters = [amp.getBBox().getCenterY()
for amp
in detector]
881 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1
882 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1
883 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int)
884 for ampId, xIndex, yIndex
in zip(np.arange(len(detector)), xIndices, yIndices):
885 ampIds[yIndex, xIndex] = ampId
889 for i, amp
in enumerate(detector):
890 y, x = np.where(ampIds == i)
891 end = ampIds.shape[1] - 1
894 thisAmpsPair = outputStats[amp.getName()][
'DIVISADERO_MAX_PAIR']
898 myMax = thisAmpsPair[1]
900 neighborMax = outputStats[detector[ampIds[yInd, 1]].getName()][
'DIVISADERO_MAX_PAIR'][0]
903 myMax = thisAmpsPair[0]
905 neighborMax = outputStats[detector[ampIds[yInd, end - 1]].getName()][
'DIVISADERO_MAX_PAIR'][1]
908 myMax = max(thisAmpsPair)
909 leftName = detector[ampIds[yInd, max(xInd - 1, 0)]].getName()
910 rightName = detector[ampIds[yInd, min(xInd + 1, ampIds.shape[1] - 1)]].getName()
913 neighborMax = max(outputStats[leftName][
'DIVISADERO_MAX_PAIR'][1],
914 outputStats[rightName][
'DIVISADERO_MAX_PAIR'][0])
916 divisaderoMax = max([myMax, neighborMax])
917 outputStats[amp.getName()][
'DIVISADERO_MAX'] = divisaderoMax