23 Apply intra-detector crosstalk corrections
26 from astropy.table
import Table
30 from lsst.pex.config
import Config, Field, ChoiceField, ListField
36 __all__ = [
"CrosstalkCalib",
"CrosstalkConfig",
"CrosstalkTask",
41 """Calibration of amp-to-amp crosstalk coefficients.
45 detector : `lsst.afw.cameraGeom.Detector`, optional
46 Detector to use to pull coefficients from.
47 nAmp : `int`, optional
48 Number of amplifiers to initialize.
49 log : `lsst.log.Log`, optional
50 Log to write messages to.
52 Parameters to pass to parent constructor.
56 The crosstalk attributes stored are:
59 Whether there is crosstalk defined for this detector.
61 Number of amplifiers in this detector.
62 crosstalkShape : `tuple` [`int`, `int`]
63 A tuple containing the shape of the ``coeffs`` matrix. This
64 should be equivalent to (``nAmp``, ``nAmp``).
66 A matrix containing the crosstalk coefficients. coeff[i][j]
67 contains the coefficients to calculate the contribution
68 amplifier_j has on amplifier_i (each row[i] contains the
69 corrections for detector_i).
70 coeffErr : `np.ndarray`, optional
71 A matrix (as defined by ``coeffs``) containing the standard
72 distribution of the crosstalk measurements.
73 coeffNum : `np.ndarray`, optional
74 A matrix containing the number of pixel pairs used to measure
75 the ``coeffs`` and ``coeffErr``.
76 coeffValid : `np.ndarray`, optional
77 A matrix of Boolean values indicating if the coefficient is
78 valid, defined as abs(coeff) > coeffErr / sqrt(coeffNum).
79 interChip : `dict` [`np.ndarray`]
80 A dictionary keyed by detectorName containing ``coeffs``
81 matrices used to correct for inter-chip crosstalk with a
82 source on the detector indicated.
85 _OBSTYPE =
'CROSSTALK'
86 _SCHEMA =
'Gen3 Crosstalk'
89 def __init__(self, detector=None, nAmp=0, **kwargs):
91 self.
nAmp = nAmp
if nAmp
else 0
97 dtype=int)
if self.
nAmp else None
99 dtype=bool)
if self.
nAmp else None
104 'coeffErr',
'coeffNum',
'coeffValid',
110 """Update calibration metadata.
112 This calls the base class's method after ensuring the required
113 calibration keywords will be saved.
117 setDate : `bool`, optional
118 Update the CALIBDATE fields in the metadata to the current
119 time. Defaults to False.
121 Other keyword parameters to set in the metadata.
127 kwargs[
'NAMP'] = self.
nAmp
134 """Set calibration parameters from the detector.
138 detector : `lsst.afw.cameraGeom.Detector`
139 Detector to use to set parameters from.
140 coeffVector : `numpy.array`, optional
141 Use the detector geometry (bounding boxes and flip
142 information), but use ``coeffVector`` instead of the
143 output of ``detector.getCrosstalk()``.
147 calib : `lsst.ip.isr.CrosstalkCalib`
148 The calibration constructed from the detector.
151 if detector.hasCrosstalk()
or coeffVector:
155 self.
nAmp = len(detector)
158 if coeffVector
is not None:
159 crosstalkCoeffs = coeffVector
161 crosstalkCoeffs = detector.getCrosstalk()
162 if len(crosstalkCoeffs) == 1
and crosstalkCoeffs[0] == 0.0:
167 raise RuntimeError(
"Crosstalk coefficients do not match detector shape. "
168 f
"{self.crosstalkShape} {self.nAmp}")
177 """Construct a calibration from a dictionary of properties.
179 Must be implemented by the specific calibration subclasses.
184 Dictionary of properties.
188 calib : `lsst.ip.isr.CalibType`
189 Constructed calibration.
194 Raised if the supplied dictionary is for a different
199 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
200 raise RuntimeError(f
"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
201 f
"found {dictionary['metadata']['OBSTYPE']}")
203 calib.setMetadata(dictionary[
'metadata'])
205 if 'detectorName' in dictionary:
206 calib._detectorName = dictionary.get(
'detectorName')
207 elif 'DETECTOR_NAME' in dictionary:
208 calib._detectorName = dictionary.get(
'DETECTOR_NAME')
209 elif 'DET_NAME' in dictionary[
'metadata']:
210 calib._detectorName = dictionary[
'metadata'][
'DET_NAME']
212 calib._detectorName =
None
214 if 'detectorSerial' in dictionary:
215 calib._detectorSerial = dictionary.get(
'detectorSerial')
216 elif 'DETECTOR_SERIAL' in dictionary:
217 calib._detectorSerial = dictionary.get(
'DETECTOR_SERIAL')
218 elif 'DET_SER' in dictionary[
'metadata']:
219 calib._detectorSerial = dictionary[
'metadata'][
'DET_SER']
221 calib._detectorSerial =
None
223 if 'detectorId' in dictionary:
224 calib._detectorId = dictionary.get(
'detectorId')
225 elif 'DETECTOR' in dictionary:
226 calib._detectorId = dictionary.get(
'DETECTOR')
227 elif 'DETECTOR' in dictionary[
'metadata']:
228 calib._detectorId = dictionary[
'metadata'][
'DETECTOR']
229 elif calib._detectorSerial:
230 calib._detectorId = calib._detectorSerial
232 calib._detectorId =
None
234 if 'instrument' in dictionary:
235 calib._instrument = dictionary.get(
'instrument')
236 elif 'INSTRUME' in dictionary[
'metadata']:
237 calib._instrument = dictionary[
'metadata'][
'INSTRUME']
239 calib._instrument =
None
241 calib.hasCrosstalk = dictionary.get(
'hasCrosstalk',
242 dictionary[
'metadata'].get(
'HAS_CROSSTALK',
False))
243 if calib.hasCrosstalk:
244 calib.nAmp = dictionary.get(
'nAmp', dictionary[
'metadata'].get(
'NAMP', 0))
245 calib.crosstalkShape = (calib.nAmp, calib.nAmp)
246 calib.coeffs = np.array(dictionary[
'coeffs']).reshape(calib.crosstalkShape)
247 if 'coeffErr' in dictionary:
248 calib.coeffErr = np.array(dictionary[
'coeffErr']).reshape(calib.crosstalkShape)
250 calib.coeffErr = np.zeros_like(calib.coeffs)
251 if 'coeffNum' in dictionary:
252 calib.coeffNum = np.array(dictionary[
'coeffNum']).reshape(calib.crosstalkShape)
254 calib.coeffNum = np.zeros_like(calib.coeffs, dtype=int)
255 if 'coeffValid' in dictionary:
256 calib.coeffValid = np.array(dictionary[
'coeffValid']).reshape(calib.crosstalkShape)
258 calib.coeffValid = np.ones_like(calib.coeffs, dtype=bool)
260 calib.interChip = dictionary.get(
'interChip',
None)
262 for detector
in calib.interChip:
263 coeffVector = calib.interChip[detector]
264 calib.interChip[detector] = np.array(coeffVector).reshape(calib.crosstalkShape)
266 calib.updateMetadata()
270 """Return a dictionary containing the calibration properties.
272 The dictionary should be able to be round-tripped through
278 Dictionary of properties.
284 outDict[
'metadata'] = metadata
287 outDict[
'nAmp'] = self.
nAmp
291 outDict[
'coeffs'] = self.
coeffs.reshape(ctLength).tolist()
294 outDict[
'coeffErr'] = self.
coeffErr.reshape(ctLength).tolist()
296 outDict[
'coeffNum'] = self.
coeffNum.reshape(ctLength).tolist()
298 outDict[
'coeffValid'] = self.
coeffValid.reshape(ctLength).tolist()
301 outDict[
'interChip'] = dict()
303 outDict[
'interChip'][detector] = self.
interChip[detector].reshape(ctLength).tolist()
309 """Construct calibration from a list of tables.
311 This method uses the `fromDict` method to create the
312 calibration, after constructing an appropriate dictionary from
317 tableList : `list` [`lsst.afw.table.Table`]
318 List of tables to use to construct the crosstalk
323 calib : `lsst.ip.isr.CrosstalkCalib`
324 The calibration defined in the tables.
327 coeffTable = tableList[0]
329 metadata = coeffTable.meta
331 inDict[
'metadata'] = metadata
332 inDict[
'hasCrosstalk'] = metadata[
'HAS_CROSSTALK']
333 inDict[
'nAmp'] = metadata[
'NAMP']
335 inDict[
'coeffs'] = coeffTable[
'CT_COEFFS']
336 if 'CT_ERRORS' in coeffTable:
337 inDict[
'coeffErr'] = coeffTable[
'CT_ERRORS']
338 if 'CT_COUNTS' in coeffTable:
339 inDict[
'coeffNum'] = coeffTable[
'CT_COUNTS']
340 if 'CT_VALID' in coeffTable:
341 inDict[
'coeffValid'] = coeffTable[
'CT_VALID']
343 if len(tableList) > 1:
344 inDict[
'interChip'] = dict()
345 interChipTable = tableList[1]
346 for record
in interChipTable:
347 inDict[
'interChip'][record[
'IC_SOURCE_DET']] = record[
'IC_COEFFS']
352 """Construct a list of tables containing the information in this calibration.
354 The list of tables should create an identical calibration
355 after being passed to this class's fromTable method.
359 tableList : `list` [`lsst.afw.table.Table`]
360 List of tables containing the crosstalk calibration
366 catalog = Table([{
'CT_COEFFS': self.
coeffs.reshape(self.
nAmp*self.
nAmp),
373 tableList.append(catalog)
376 interChipTable = Table([{
'IC_SOURCE_DET': sourceDet,
379 tableList.append(interChipTable)
385 """Extract the image data from an amp, flipped to match ampTarget.
389 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
390 Image containing the amplifier of interest.
391 amp : `lsst.afw.cameraGeom.Amplifier`
392 Amplifier on image to extract.
393 ampTarget : `lsst.afw.cameraGeom.Amplifier`
394 Target amplifier that the extracted image will be flipped
397 The image is already trimmed.
398 TODO : DM-15409 will resolve this.
402 output : `lsst.afw.image.Image`
403 Image of the amplifier in the desired configuration.
405 X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
406 lsst.afw.cameraGeom.ReadoutCorner.LR:
True,
407 lsst.afw.cameraGeom.ReadoutCorner.UL:
False,
408 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
409 Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
410 lsst.afw.cameraGeom.ReadoutCorner.LR:
False,
411 lsst.afw.cameraGeom.ReadoutCorner.UL:
True,
412 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
414 output = image[amp.getBBox()
if isTrimmed
else amp.getRawDataBBox()]
415 thisAmpCorner = amp.getReadoutCorner()
416 targetAmpCorner = ampTarget.getReadoutCorner()
419 xFlip = X_FLIP[targetAmpCorner] ^ X_FLIP[thisAmpCorner]
420 yFlip = Y_FLIP[targetAmpCorner] ^ Y_FLIP[thisAmpCorner]
425 """Estimate median background in image.
427 Getting a great background model isn't important for crosstalk correction,
428 since the crosstalk is at a low level. The median should be sufficient.
432 mi : `lsst.afw.image.MaskedImage`
433 MaskedImage for which to measure background.
434 badPixels : `list` of `str`
435 Mask planes to ignore.
439 Median background level.
443 stats.setAndMask(mask.getPlaneBitMask(badPixels))
447 badPixels=["BAD"], minPixelToMask=45000,
448 crosstalkStr="CROSSTALK", isTrimmed=False,
449 backgroundMethod="None"):
450 """Subtract the crosstalk from thisExposure, optionally using a different source.
452 We set the mask plane indicated by ``crosstalkStr`` in a target amplifier
453 for pixels in a source amplifier that exceed ``minPixelToMask``. Note that
454 the correction is applied to all pixels in the amplifier, but only those
455 that have a substantial crosstalk are masked with ``crosstalkStr``.
457 The uncorrected image is used as a template for correction. This is good
458 enough if the crosstalk is small (e.g., coefficients < ~ 1e-3), but if it's
459 larger you may want to iterate.
463 thisExposure : `lsst.afw.image.Exposure`
464 Exposure for which to subtract crosstalk.
465 sourceExposure : `lsst.afw.image.Exposure`, optional
466 Exposure to use as the source of the crosstalk. If not set,
467 thisExposure is used as the source (intra-detector crosstalk).
468 crosstalkCoeffs : `numpy.ndarray`, optional.
469 Coefficients to use to correct crosstalk.
470 badPixels : `list` of `str`
471 Mask planes to ignore.
472 minPixelToMask : `float`
473 Minimum pixel value (relative to the background level) in
474 source amplifier for which to set ``crosstalkStr`` mask plane
477 Mask plane name for pixels greatly modified by crosstalk
478 (above minPixelToMask).
480 The image is already trimmed.
481 This should no longer be needed once DM-15409 is resolved.
482 backgroundMethod : `str`
483 Method used to subtract the background. "AMP" uses
484 amplifier-by-amplifier background levels, "DETECTOR" uses full
485 exposure/maskedImage levels. Any other value results in no
486 background subtraction.
488 mi = thisExposure.getMaskedImage()
490 detector = thisExposure.getDetector()
492 self.
fromDetector(detector, coeffVector=crosstalkCoeffs)
494 numAmps = len(detector)
495 if numAmps != self.
nAmp:
496 raise RuntimeError(f
"Crosstalk built for {self.nAmp} in {self._detectorName}, received "
497 f
"{numAmps} in {detector.getName()}")
500 source = sourceExposure.getMaskedImage()
501 sourceDetector = sourceExposure.getDetector()
504 sourceDetector = detector
506 if crosstalkCoeffs
is not None:
507 coeffs = crosstalkCoeffs
510 self.
log.debug(
"CT COEFF: %s", coeffs)
517 backgrounds = [0.0
for amp
in sourceDetector]
518 if backgroundMethod
is None:
520 elif backgroundMethod ==
"AMP":
522 for amp
in sourceDetector]
523 elif backgroundMethod ==
"DETECTOR":
528 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
531 + thresholdBackground))
532 footprints.setMask(mask, crosstalkStr)
533 crosstalk = mask.getPlaneBitMask(crosstalkStr)
536 subtrahend = source.Factory(source.getBBox())
537 subtrahend.set((0, 0, 0))
539 coeffs = coeffs.transpose()
540 for ii, iAmp
in enumerate(sourceDetector):
541 iImage = subtrahend[iAmp.getBBox()
if isTrimmed
else iAmp.getRawDataBBox()]
542 for jj, jAmp
in enumerate(detector):
543 if coeffs[ii, jj] == 0.0:
545 jImage = self.
extractAmp(mi, jAmp, iAmp, isTrimmed)
546 jImage.getMask().getArray()[:] &= crosstalk
547 jImage -= backgrounds[jj]
548 iImage.scaledPlus(coeffs[ii, jj], jImage)
552 mask.clearMaskPlane(crosstalkPlane)
557 """Configuration for intra-detector crosstalk removal."""
558 minPixelToMask = Field(
560 doc=
"Set crosstalk mask plane for pixels over this value.",
563 crosstalkMaskPlane = Field(
565 doc=
"Name for crosstalk mask plane.",
568 crosstalkBackgroundMethod = ChoiceField(
570 doc=
"Type of background subtraction to use when applying correction.",
573 "None":
"Do no background subtraction.",
574 "AMP":
"Subtract amplifier-by-amplifier background levels.",
575 "DETECTOR":
"Subtract detector level background."
578 useConfigCoefficients = Field(
580 doc=
"Ignore the detector crosstalk information in favor of CrosstalkConfig values?",
583 crosstalkValues = ListField(
585 doc=(
"Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 "
586 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. "
587 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column "
588 "vector [corr0 corr1 corr2 ...]^T."),
591 crosstalkShape = ListField(
593 doc=
"Shape of the coefficient array. This should be equal to [nAmp, nAmp].",
598 """Return a 2-D numpy array of crosstalk coefficients in the proper shape.
602 detector : `lsst.afw.cameraGeom.detector`
603 Detector that is to be crosstalk corrected.
607 coeffs : `numpy.ndarray`
608 Crosstalk coefficients that can be used to correct the detector.
613 Raised if no coefficients could be generated from this detector/configuration.
617 if detector
is not None:
619 if coeffs.shape != (nAmp, nAmp):
620 raise RuntimeError(
"Constructed crosstalk coeffients do not match detector shape. "
621 f
"{coeffs.shape} {nAmp}")
623 elif detector
is not None and detector.hasCrosstalk()
is True:
625 return detector.getCrosstalk()
627 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients")
630 """Return a boolean indicating if crosstalk coefficients exist.
634 detector : `lsst.afw.cameraGeom.detector`
635 Detector that is to be crosstalk corrected.
639 hasCrosstalk : `bool`
640 True if this detector/configuration has crosstalk coefficients defined.
644 elif detector
is not None and detector.hasCrosstalk()
is True:
651 """Apply intra-detector crosstalk correction."""
652 ConfigClass = CrosstalkConfig
653 _DefaultName =
'isrCrosstalk'
656 """Placeholder for crosstalk preparation method, e.g., for inter-detector crosstalk.
660 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
661 Butler reference of the detector data to be processed.
662 crosstalk : `~lsst.ip.isr.CrosstalkConfig`
663 Crosstalk calibration that will be used.
667 lsst.obs.decam.crosstalk.DecamCrosstalkTask.prepCrosstalk
671 def run(self, exposure, crosstalk=None,
672 crosstalkSources=None, isTrimmed=False):
673 """Apply intra-detector crosstalk correction
677 exposure : `lsst.afw.image.Exposure`
678 Exposure for which to remove crosstalk.
679 crosstalkCalib : `lsst.ip.isr.CrosstalkCalib`, optional
680 External crosstalk calibration to apply. Constructed from
681 detector if not found.
682 crosstalkSources : `defaultdict`, optional
683 Image data for other detectors that are sources of
684 crosstalk in exposure. The keys are expected to be names
685 of the other detectors, with the values containing
686 `lsst.afw.image.Exposure` at the same level of processing
688 The default for intra-detector crosstalk here is None.
690 The image is already trimmed.
691 This should no longer be needed once DM-15409 is resolved.
696 Raised if called for a detector that does not have a
697 crosstalk correction.
701 crosstalk = crosstalk.fromDetector(exposure.getDetector(),
702 coeffVector=self.config.crosstalkValues)
703 if not crosstalk.log:
704 crosstalk.log = self.log
705 if not crosstalk.hasCrosstalk:
706 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients.")
709 self.log.info(
"Applying crosstalk correction.")
710 crosstalk.subtractCrosstalk(exposure, crosstalkCoeffs=crosstalk.coeffs,
711 minPixelToMask=self.config.minPixelToMask,
712 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed,
713 backgroundMethod=self.config.crosstalkBackgroundMethod)
715 if crosstalk.interChip:
717 for detName
in crosstalk.interChip:
718 if isinstance(crosstalkSources[0],
'lsst.afw.image.Exposure'):
720 sourceNames = [exp.getDetector().getName()
for exp
in crosstalkSources]
723 sourceNames = [expRef.get(datasetType=
'isrOscanCorr').getDetector().getName()
724 for expRef
in crosstalkSources]
725 if detName
not in sourceNames:
726 self.log.warn(
"Crosstalk lists %s, not found in sources: %s",
727 detName, sourceNames)
729 interChipCoeffs = crosstalk.interChip[detName]
730 sourceExposure = crosstalkSources[sourceNames.index(detName)]
731 crosstalk.subtractCrosstalk(exposure, sourceExposure=sourceExposure,
732 crosstalkCoeffs=interChipCoeffs,
733 minPixelToMask=self.config.minPixelToMask,
734 crosstalkStr=self.config.crosstalkMaskPlane,
736 backgroundMethod=self.config.crosstalkBackgroundMethod)
738 self.log.warn(
"Crosstalk contains interChip coefficients, but no sources found!")
742 def run(self, exposure, crosstalkSources=None):
743 self.log.info(
"Not performing any crosstalk correction")