23Apply intra-detector crosstalk corrections
26__all__ = [
"CrosstalkCalib",
"CrosstalkConfig",
"CrosstalkTask",
30from astropy.table
import Table
36from lsst.pipe.base
import Task
42 """Calibration of amp-to-amp crosstalk coefficients.
47 Detector to use to pull coefficients from.
48 nAmp : `int`, optional
49 Number of amplifiers to initialize.
50 log : `logging.Logger`, optional
51 Log to write messages to.
53 Parameters to
pass to parent constructor.
57 The crosstalk attributes stored are:
60 Whether there
is crosstalk defined
for this detector.
62 Number of amplifiers
in this detector.
63 crosstalkShape : `tuple` [`int`, `int`]
64 A tuple containing the shape of the ``coeffs`` matrix. This
65 should be equivalent to (``nAmp``, ``nAmp``).
66 coeffs : `numpy.ndarray`
67 A matrix containing the crosstalk coefficients. coeff[i][j]
68 contains the coefficients to calculate the contribution
69 amplifier_j has on amplifier_i (each row[i] contains the
70 corrections
for detector_i).
71 coeffErr : `numpy.ndarray`, optional
72 A matrix (
as defined by ``coeffs``) containing the standard
73 distribution of the crosstalk measurements.
74 coeffNum : `numpy.ndarray`, optional
75 A matrix containing the number of pixel pairs used to measure
76 the ``coeffs``
and ``coeffErr``.
77 coeffValid : `numpy.ndarray`, optional
78 A matrix of Boolean values indicating
if the coefficient
is
79 valid, defined
as abs(coeff) > coeffErr / sqrt(coeffNum).
80 interChip : `dict` [`numpy.ndarray`]
81 A dictionary keyed by detectorName containing ``coeffs``
82 matrices used to correct
for inter-chip crosstalk
with a
83 source on the detector indicated.
86 _OBSTYPE = 'CROSSTALK'
87 _SCHEMA =
'Gen3 Crosstalk'
90 def __init__(self, detector=None, nAmp=0, **kwargs):
92 self.
nAmp = nAmp
if nAmp
else 0
98 dtype=int)
if self.
nAmp else None
100 dtype=bool)
if self.
nAmp else None
105 'coeffErr',
'coeffNum',
'coeffValid',
111 """Update calibration metadata.
113 This calls the base class's method after ensuring the required
114 calibration keywords will be saved.
118 setDate : `bool`, optional
119 Update the CALIBDATE fields in the metadata to the current
120 time. Defaults to
False.
122 Other keyword parameters to set
in the metadata.
128 kwargs[
'NAMP'] = self.
nAmp
135 """Set calibration parameters from the detector.
140 Detector to use to set parameters from.
141 coeffVector : `numpy.array`, optional
142 Use the detector geometry (bounding boxes
and flip
143 information), but use ``coeffVector`` instead of the
144 output of ``detector.getCrosstalk()``.
149 The calibration constructed
from the detector.
152 if detector.hasCrosstalk()
or coeffVector:
157 self.
nAmp = len(detector)
160 if coeffVector
is not None:
161 crosstalkCoeffs = coeffVector
163 crosstalkCoeffs = detector.getCrosstalk()
164 if len(crosstalkCoeffs) == 1
and crosstalkCoeffs[0] == 0.0:
169 raise RuntimeError(
"Crosstalk coefficients do not match detector shape. "
170 f
"{self.crosstalkShape} {self.nAmp}")
183 """Construct a calibration from a dictionary of properties.
185 Must be implemented by the specific calibration subclasses.
190 Dictionary of properties.
194 calib : `lsst.ip.isr.CalibType`
195 Constructed calibration.
200 Raised if the supplied dictionary
is for a different
205 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
206 raise RuntimeError(f
"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
207 f
"found {dictionary['metadata']['OBSTYPE']}")
209 calib.setMetadata(dictionary[
'metadata'])
211 if 'detectorName' in dictionary:
212 calib._detectorName = dictionary.get(
'detectorName')
213 elif 'DETECTOR_NAME' in dictionary:
214 calib._detectorName = dictionary.get(
'DETECTOR_NAME')
215 elif 'DET_NAME' in dictionary[
'metadata']:
216 calib._detectorName = dictionary[
'metadata'][
'DET_NAME']
218 calib._detectorName =
None
220 if 'detectorSerial' in dictionary:
221 calib._detectorSerial = dictionary.get(
'detectorSerial')
222 elif 'DETECTOR_SERIAL' in dictionary:
223 calib._detectorSerial = dictionary.get(
'DETECTOR_SERIAL')
224 elif 'DET_SER' in dictionary[
'metadata']:
225 calib._detectorSerial = dictionary[
'metadata'][
'DET_SER']
227 calib._detectorSerial =
None
229 if 'detectorId' in dictionary:
230 calib._detectorId = dictionary.get(
'detectorId')
231 elif 'DETECTOR' in dictionary:
232 calib._detectorId = dictionary.get(
'DETECTOR')
233 elif 'DETECTOR' in dictionary[
'metadata']:
234 calib._detectorId = dictionary[
'metadata'][
'DETECTOR']
235 elif calib._detectorSerial:
236 calib._detectorId = calib._detectorSerial
238 calib._detectorId =
None
240 if 'instrument' in dictionary:
241 calib._instrument = dictionary.get(
'instrument')
242 elif 'INSTRUME' in dictionary[
'metadata']:
243 calib._instrument = dictionary[
'metadata'][
'INSTRUME']
245 calib._instrument =
None
247 calib.hasCrosstalk = dictionary.get(
'hasCrosstalk',
248 dictionary[
'metadata'].get(
'HAS_CROSSTALK',
False))
249 if calib.hasCrosstalk:
250 calib.nAmp = dictionary.get(
'nAmp', dictionary[
'metadata'].get(
'NAMP', 0))
251 calib.crosstalkShape = (calib.nAmp, calib.nAmp)
252 calib.coeffs = np.array(dictionary[
'coeffs']).reshape(calib.crosstalkShape)
253 if 'coeffErr' in dictionary:
254 calib.coeffErr = np.array(dictionary[
'coeffErr']).reshape(calib.crosstalkShape)
256 calib.coeffErr = np.zeros_like(calib.coeffs)
257 if 'coeffNum' in dictionary:
258 calib.coeffNum = np.array(dictionary[
'coeffNum']).reshape(calib.crosstalkShape)
260 calib.coeffNum = np.zeros_like(calib.coeffs, dtype=int)
261 if 'coeffValid' in dictionary:
262 calib.coeffValid = np.array(dictionary[
'coeffValid']).reshape(calib.crosstalkShape)
264 calib.coeffValid = np.ones_like(calib.coeffs, dtype=bool)
266 calib.interChip = dictionary.get(
'interChip',
None)
268 for detector
in calib.interChip:
269 coeffVector = calib.interChip[detector]
270 calib.interChip[detector] = np.array(coeffVector).reshape(calib.crosstalkShape)
272 calib.updateMetadata()
276 """Return a dictionary containing the calibration properties.
278 The dictionary should be able to be round-tripped through
284 Dictionary of properties.
290 outDict['metadata'] = metadata
293 outDict[
'nAmp'] = self.
nAmp
297 outDict[
'coeffs'] = self.
coeffs.reshape(ctLength).tolist()
300 outDict[
'coeffErr'] = self.
coeffErr.reshape(ctLength).tolist()
302 outDict[
'coeffNum'] = self.
coeffNum.reshape(ctLength).tolist()
304 outDict[
'coeffValid'] = self.
coeffValid.reshape(ctLength).tolist()
307 outDict[
'interChip'] = dict()
309 outDict[
'interChip'][detector] = self.
interChip[detector].reshape(ctLength).tolist()
315 """Construct calibration from a list of tables.
317 This method uses the `fromDict` method to create the
318 calibration, after constructing an appropriate dictionary from
323 tableList : `list` [`lsst.afw.table.Table`]
324 List of tables to use to construct the crosstalk
330 The calibration defined
in the tables.
333 coeffTable = tableList[0]
335 metadata = coeffTable.meta
337 inDict['metadata'] = metadata
338 inDict[
'hasCrosstalk'] = metadata[
'HAS_CROSSTALK']
339 inDict[
'nAmp'] = metadata[
'NAMP']
341 inDict[
'coeffs'] = coeffTable[
'CT_COEFFS']
342 if 'CT_ERRORS' in coeffTable.columns:
343 inDict[
'coeffErr'] = coeffTable[
'CT_ERRORS']
344 if 'CT_COUNTS' in coeffTable.columns:
345 inDict[
'coeffNum'] = coeffTable[
'CT_COUNTS']
346 if 'CT_VALID' in coeffTable.columns:
347 inDict[
'coeffValid'] = coeffTable[
'CT_VALID']
349 if len(tableList) > 1:
350 inDict[
'interChip'] = dict()
351 interChipTable = tableList[1]
352 for record
in interChipTable:
353 inDict[
'interChip'][record[
'IC_SOURCE_DET']] = record[
'IC_COEFFS']
358 """Construct a list of tables containing the information in this
361 The list of tables should create an identical calibration
362 after being passed to this class's fromTable method.
366 tableList : `list` [`lsst.afw.table.Table`]
367 List of tables containing the crosstalk calibration
373 catalog = Table([{'CT_COEFFS': self.
coeffs.reshape(self.
nAmp*self.
nAmp),
380 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
381 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
382 catalog.meta = outMeta
383 tableList.append(catalog)
386 interChipTable = Table([{
'IC_SOURCE_DET': sourceDet,
389 tableList.append(interChipTable)
395 """Extract the image data from an amp, flipped to match ampTarget.
400 Image containing the amplifier of interest.
402 Amplifier on image to extract.
404 Target amplifier that the extracted image will be flipped
407 The image
is already trimmed.
408 TODO : DM-15409 will resolve this.
413 Image of the amplifier
in the desired configuration.
415 X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False,
416 lsst.afw.cameraGeom.ReadoutCorner.LR:
True,
417 lsst.afw.cameraGeom.ReadoutCorner.UL:
False,
418 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
419 Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
420 lsst.afw.cameraGeom.ReadoutCorner.LR:
False,
421 lsst.afw.cameraGeom.ReadoutCorner.UL:
True,
422 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
424 output = image[amp.getBBox()
if isTrimmed
else amp.getRawDataBBox()]
425 thisAmpCorner = amp.getReadoutCorner()
426 targetAmpCorner = ampTarget.getReadoutCorner()
430 xFlip = X_FLIP[targetAmpCorner] ^ X_FLIP[thisAmpCorner]
431 yFlip = Y_FLIP[targetAmpCorner] ^ Y_FLIP[thisAmpCorner]
436 """Estimate median background in image.
438 Getting a great background model isn't important for crosstalk
439 correction, since the crosstalk is at a low level. The median should
445 MaskedImage
for which to measure background.
446 badPixels : `list` of `str`
447 Mask planes to ignore.
451 Median background level.
455 stats.setAndMask(mask.getPlaneBitMask(badPixels))
459 badPixels=["BAD"], minPixelToMask=45000,
460 crosstalkStr="CROSSTALK", isTrimmed=False,
461 backgroundMethod="None"):
462 """Subtract the crosstalk from thisExposure, optionally using a
465 We set the mask plane indicated by ``crosstalkStr`` in a target
466 amplifier
for pixels
in a source amplifier that exceed
467 ``minPixelToMask``. Note that the correction
is applied to all pixels
468 in the amplifier, but only those that have a substantial crosstalk
469 are masked
with ``crosstalkStr``.
471 The uncorrected image
is used
as a template
for correction. This
is
472 good enough
if the crosstalk
is small (e.g., coefficients < ~ 1e-3),
473 but
if it
's larger you may want to iterate.
478 Exposure for which to subtract crosstalk.
480 Exposure to use
as the source of the crosstalk. If
not set,
481 thisExposure
is used
as the source (intra-detector crosstalk).
482 crosstalkCoeffs : `numpy.ndarray`, optional.
483 Coefficients to use to correct crosstalk.
484 badPixels : `list` of `str`
485 Mask planes to ignore.
486 minPixelToMask : `float`
487 Minimum pixel value (relative to the background level)
in
488 source amplifier
for which to set ``crosstalkStr`` mask plane
491 Mask plane name
for pixels greatly modified by crosstalk
492 (above minPixelToMask).
494 The image
is already trimmed.
495 This should no longer be needed once DM-15409
is resolved.
496 backgroundMethod : `str`
497 Method used to subtract the background.
"AMP" uses
498 amplifier-by-amplifier background levels,
"DETECTOR" uses full
499 exposure/maskedImage levels. Any other value results
in no
500 background subtraction.
502 mi = thisExposure.getMaskedImage()
504 detector = thisExposure.getDetector()
508 numAmps = len(detector)
509 if numAmps != self.
nAmp:
510 raise RuntimeError(f
"Crosstalk built for {self.nAmp} in {self._detectorName}, received "
511 f
"{numAmps} in {detector.getName()}")
514 source = sourceExposure.getMaskedImage()
515 sourceDetector = sourceExposure.getDetector()
518 sourceDetector = detector
520 if crosstalkCoeffs
is not None:
521 coeffs = crosstalkCoeffs
524 self.
log.debug(
"CT COEFF: %s", coeffs)
531 backgrounds = [0.0
for amp
in sourceDetector]
532 if backgroundMethod
is None:
534 elif backgroundMethod ==
"AMP":
536 for amp
in sourceDetector]
537 elif backgroundMethod ==
"DETECTOR":
542 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
545 + thresholdBackground))
546 footprints.setMask(mask, crosstalkStr)
547 crosstalk = mask.getPlaneBitMask(crosstalkStr)
550 subtrahend = source.Factory(source.getBBox())
551 subtrahend.set((0, 0, 0))
553 coeffs = coeffs.transpose()
554 for ii, iAmp
in enumerate(sourceDetector):
555 iImage = subtrahend[iAmp.getBBox()
if isTrimmed
else iAmp.getRawDataBBox()]
556 for jj, jAmp
in enumerate(detector):
557 if coeffs[ii, jj] == 0.0:
559 jImage = self.
extractAmp(mi, jAmp, iAmp, isTrimmed)
560 jImage.getMask().getArray()[:] &= crosstalk
561 jImage -= backgrounds[jj]
562 iImage.scaledPlus(coeffs[ii, jj], jImage)
567 mask.clearMaskPlane(crosstalkPlane)
572 """Configuration for intra-detector crosstalk removal."""
573 minPixelToMask = Field(
575 doc=
"Set crosstalk mask plane for pixels over this value.",
578 crosstalkMaskPlane = Field(
580 doc=
"Name for crosstalk mask plane.",
583 crosstalkBackgroundMethod = ChoiceField(
585 doc=
"Type of background subtraction to use when applying correction.",
588 "None":
"Do no background subtraction.",
589 "AMP":
"Subtract amplifier-by-amplifier background levels.",
590 "DETECTOR":
"Subtract detector level background."
593 useConfigCoefficients = Field(
595 doc=
"Ignore the detector crosstalk information in favor of CrosstalkConfig values?",
598 crosstalkValues = ListField(
600 doc=(
"Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 "
601 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. "
602 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column "
603 "vector [corr0 corr1 corr2 ...]^T."),
606 crosstalkShape = ListField(
608 doc=
"Shape of the coefficient array. This should be equal to [nAmp, nAmp].",
613 """Return a 2-D numpy array of crosstalk coefficients in the proper
618 detector : `lsst.afw.cameraGeom.detector`
619 Detector that is to be crosstalk corrected.
623 coeffs : `numpy.ndarray`
624 Crosstalk coefficients that can be used to correct the detector.
629 Raised
if no coefficients could be generated
from this
630 detector/configuration.
634 if detector
is not None:
636 if coeffs.shape != (nAmp, nAmp):
637 raise RuntimeError(
"Constructed crosstalk coeffients do not match detector shape. "
638 f
"{coeffs.shape} {nAmp}")
640 elif detector
is not None and detector.hasCrosstalk()
is True:
642 return detector.getCrosstalk()
644 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients")
647 """Return a boolean indicating if crosstalk coefficients exist.
651 detector : `lsst.afw.cameraGeom.detector`
652 Detector that is to be crosstalk corrected.
656 hasCrosstalk : `bool`
657 True if this detector/configuration has crosstalk coefficients
662 elif detector
is not None and detector.hasCrosstalk()
is True:
669 """Apply intra-detector crosstalk correction."""
670 ConfigClass = CrosstalkConfig
671 _DefaultName =
'isrCrosstalk'
673 def run(self, exposure, crosstalk=None,
674 crosstalkSources=None, isTrimmed=False, camera=None):
675 """Apply intra-detector crosstalk correction
680 Exposure for which to remove crosstalk.
682 External crosstalk calibration to apply. Constructed
from
683 detector
if not found.
684 crosstalkSources : `defaultdict`, optional
685 Image data
for other detectors that are sources of
686 crosstalk
in exposure. The keys are expected to be names
687 of the other detectors,
with the values containing
690 The default
for intra-detector crosstalk here
is None.
691 isTrimmed : `bool`, optional
692 The image
is already trimmed.
693 This should no longer be needed once DM-15409
is resolved.
695 Camera associated
with this exposure. Only used
for
701 Raised
if called
for a detector that does
not have a
702 crosstalk correction. Also raised
if the crosstalkSource
703 is not an expected type.
707 crosstalk = crosstalk.fromDetector(exposure.getDetector(),
708 coeffVector=self.config.crosstalkValues)
709 if not crosstalk.log:
710 crosstalk.log = self.log
711 if not crosstalk.hasCrosstalk:
712 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients.")
715 self.log.info(
"Applying crosstalk correction.")
716 crosstalk.subtractCrosstalk(exposure, crosstalkCoeffs=crosstalk.coeffs,
717 minPixelToMask=self.config.minPixelToMask,
718 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed,
719 backgroundMethod=self.config.crosstalkBackgroundMethod)
721 if crosstalk.interChip:
727 sourceNames = [exp.getDetector().getName()
for exp
in crosstalkSources]
728 elif isinstance(crosstalkSources[0], lsst.daf.butler.DeferredDatasetHandle):
730 detectorList = [source.dataId[
'detector']
for source
in crosstalkSources]
731 sourceNames = [camera[detector].getName()
for detector
in detectorList]
733 raise RuntimeError(
"Unknown object passed as crosstalk sources.",
734 type(crosstalkSources[0]))
736 for detName
in crosstalk.interChip:
737 if detName
not in sourceNames:
738 self.log.warning(
"Crosstalk lists %s, not found in sources: %s",
739 detName, sourceNames)
742 interChipCoeffs = crosstalk.interChip[detName]
744 sourceExposure = crosstalkSources[sourceNames.index(detName)]
745 if isinstance(sourceExposure, lsst.daf.butler.DeferredDatasetHandle):
747 sourceExposure = sourceExposure.get()
749 raise RuntimeError(
"Unknown object passed as crosstalk sources.",
750 type(sourceExposure))
752 self.log.info(
"Correcting detector %s with ctSource %s",
753 exposure.getDetector().getName(),
754 sourceExposure.getDetector().getName())
755 crosstalk.subtractCrosstalk(exposure, sourceExposure=sourceExposure,
756 crosstalkCoeffs=interChipCoeffs,
757 minPixelToMask=self.config.minPixelToMask,
758 crosstalkStr=self.config.crosstalkMaskPlane,
760 backgroundMethod=self.config.crosstalkBackgroundMethod)
762 self.log.warning(
"Crosstalk contains interChip coefficients, but no sources found!")
766 def run(self, exposure, crosstalkSources=None):
767 self.log.info(
"Not performing any crosstalk correction")
def requiredAttributes(self, value)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDetector(self, detector)
def requiredAttributes(self)
def __init__(self, detector=None, nAmp=0, **kwargs)
def calculateBackground(mi, badPixels=["BAD"])
def fromTable(cls, tableList)
def fromDetector(self, detector, coeffVector=None)
def updateMetadata(self, setDate=False, **kwargs)
def subtractCrosstalk(self, thisExposure, sourceExposure=None, crosstalkCoeffs=None, badPixels=["BAD"], minPixelToMask=45000, crosstalkStr="CROSSTALK", isTrimmed=False, backgroundMethod="None")
def extractAmp(image, amp, ampTarget, isTrimmed=False)
def fromDict(cls, dictionary)
def hasCrosstalk(self, detector=None)
def getCrosstalk(self, detector=None)
def run(self, exposure, crosstalk=None, crosstalkSources=None, isTrimmed=False, camera=None)
def run(self, exposure, crosstalkSources=None)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
std::shared_ptr< ImageT > flipImage(ImageT const &inImage, bool flipLR, bool flipTB)