Coverage for python/lsst/ip/isr/crosstalk.py: 9%
400 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 04:05 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-02 04:05 -0700
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Apply intra-detector crosstalk corrections
24"""
26__all__ = ["CrosstalkCalib", "CrosstalkConfig", "CrosstalkTask",
27 "NullCrosstalkTask"]
29import numpy as np
30from astropy.table import Table
32import lsst.afw.math
33import lsst.afw.detection
34import lsst.daf.butler
35from lsst.pex.config import Config, Field, ChoiceField, ListField
36from lsst.pipe.base import Task
38from lsst.ip.isr import IsrCalib
41class CrosstalkCalib(IsrCalib):
42 """Calibration of amp-to-amp crosstalk coefficients.
44 Parameters
45 ----------
46 detector : `lsst.afw.cameraGeom.Detector`, optional
47 Detector to use to pull coefficients from.
48 nAmp : `int`, optional
49 Number of amplifiers to initialize.
50 log : `logging.Logger`, optional
51 Log to write messages to.
52 **kwargs :
53 Parameters to pass to parent constructor.
55 Notes
56 -----
57 The crosstalk attributes stored are:
59 hasCrosstalk : `bool`
60 Whether there is crosstalk defined for this detector.
61 nAmp : `int`
62 Number of amplifiers in this detector.
63 crosstalkShape : `tuple` [`int`, `int`]
64 A tuple containing the shape of the ``coeffs`` matrix. This
65 should be equivalent to (``nAmp``, ``nAmp``).
66 coeffs : `numpy.ndarray`
67 A matrix containing the crosstalk coefficients. coeff[i][j]
68 contains the coefficients to calculate the contribution
69 amplifier_j has on amplifier_i (each row[i] contains the
70 corrections for detector_i).
71 coeffErr : `numpy.ndarray`, optional
72 A matrix (as defined by ``coeffs``) containing the standard
73 distribution of the crosstalk measurements.
74 coeffNum : `numpy.ndarray`, optional
75 A matrix containing the number of pixel pairs used to measure
76 the ``coeffs`` and ``coeffErr``.
77 coeffValid : `numpy.ndarray`, optional
78 A matrix of Boolean values indicating if the coefficient is
79 valid, defined as abs(coeff) > coeffErr / sqrt(coeffNum).
80 coeffsSqr : `numpy.ndarray`, optional
81 A matrix containing potential quadratic crosstalk coefficients
82 (see e.g., Snyder+21, 2001.03223). coeffsSqr[i][j]
83 contains the coefficients to calculate the contribution
84 amplifier_j has on amplifier_i (each row[i] contains the
85 corrections for detector_i).
86 coeffErrSqr : `numpy.ndarray`, optional
87 A matrix (as defined by ``coeffsSqr``) containing the standard
88 distribution of the quadratic term of the crosstalk measurements.
89 interChip : `dict` [`numpy.ndarray`]
90 A dictionary keyed by detectorName containing ``coeffs``
91 matrices used to correct for inter-chip crosstalk with a
92 source on the detector indicated.
94 Version 1.1 adds quadratic coefficients, a matrix with the ratios
95 of amplifiers gains per detector, and a field to indicate the units
96 of the numerator and denominator of the source and target signals, with
97 "adu" meaning "ADU / ADU" and "electron" meaning "e- / e-".
98 """
99 _OBSTYPE = 'CROSSTALK'
100 _SCHEMA = 'Gen3 Crosstalk'
101 _VERSION = 1.1
103 def __init__(self, detector=None, nAmp=0, **kwargs):
104 self.hasCrosstalk = False
105 self.nAmp = nAmp if nAmp else 0
106 self.crosstalkShape = (self.nAmp, self.nAmp)
108 self.coeffs = np.zeros(self.crosstalkShape) if self.nAmp else None
109 self.coeffErr = np.zeros(self.crosstalkShape) if self.nAmp else None
110 self.coeffNum = np.zeros(self.crosstalkShape,
111 dtype=int) if self.nAmp else None
112 self.coeffValid = np.zeros(self.crosstalkShape,
113 dtype=bool) if self.nAmp else None
114 # Quadratic terms, if any.
115 self.coeffsSqr = np.zeros(self.crosstalkShape) if self.nAmp else None
116 self.coeffErrSqr = np.zeros(self.crosstalkShape) if self.nAmp else None
118 # Gain ratios
119 self.ampGainRatios = np.zeros(self.crosstalkShape) if self.nAmp else None
121 # Units
122 self.crosstalkRatiosUnits = 'adu' if self.nAmp else None
124 self.interChip = {}
126 super().__init__(**kwargs)
127 self.requiredAttributes.update(['hasCrosstalk', 'nAmp', 'coeffs',
128 'coeffErr', 'coeffNum', 'coeffValid',
129 'coeffsSqr', 'coeffErrSqr',
130 'ampGainRatios', 'crosstalkRatiosUnits',
131 'interChip'])
132 if detector:
133 self.fromDetector(detector)
135 def updateMetadata(self, setDate=False, **kwargs):
136 """Update calibration metadata.
138 This calls the base class's method after ensuring the required
139 calibration keywords will be saved.
141 Parameters
142 ----------
143 setDate : `bool`, optional
144 Update the CALIBDATE fields in the metadata to the current
145 time. Defaults to False.
146 kwargs :
147 Other keyword parameters to set in the metadata.
148 """
149 kwargs['DETECTOR'] = self._detectorId
150 kwargs['DETECTOR_NAME'] = self._detectorName
151 kwargs['DETECTOR_SERIAL'] = self._detectorSerial
152 kwargs['HAS_CROSSTALK'] = self.hasCrosstalk
153 kwargs['NAMP'] = self.nAmp
154 self.crosstalkShape = (self.nAmp, self.nAmp)
155 kwargs['CROSSTALK_SHAPE'] = self.crosstalkShape
156 kwargs['CROSSTALK_RATIOS_UNITS'] = self.crosstalkRatiosUnits
158 super().updateMetadata(setDate=setDate, **kwargs)
160 def fromDetector(self, detector, coeffVector=None, coeffSqrVector=None):
161 """Set calibration parameters from the detector.
163 Parameters
164 ----------
165 detector : `lsst.afw.cameraGeom.Detector`
166 Detector to use to set parameters from.
167 coeffVector : `numpy.array`, optional
168 Use the detector geometry (bounding boxes and flip
169 information), but use ``coeffVector`` instead of the
170 output of ``detector.getCrosstalk()``.
171 coeffSqrVector : `numpy.array`, optional
172 Quadratic crosstalk coefficients.
174 Returns
175 -------
176 calib : `lsst.ip.isr.CrosstalkCalib`
177 The calibration constructed from the detector.
179 """
180 self._detectorId = detector.getId()
181 self._detectorName = detector.getName()
182 self._detectorSerial = detector.getSerial()
184 self.nAmp = len(detector)
185 self.crosstalkShape = (self.nAmp, self.nAmp)
187 if coeffVector is not None:
188 crosstalkCoeffs = coeffVector
189 else:
190 crosstalkCoeffs = detector.getCrosstalk()
191 if coeffSqrVector is not None:
192 self.coeffsSqr = coeffSqrVector
193 else:
194 self.coeffsSqr = np.zeros(self.crosstalkShape)
195 if len(crosstalkCoeffs) == 1 and crosstalkCoeffs[0] == 0.0:
196 return self
197 self.coeffs = np.array(crosstalkCoeffs).reshape(self.crosstalkShape)
199 if self.coeffs.shape != self.crosstalkShape:
200 raise RuntimeError("Crosstalk coefficients do not match detector shape. "
201 f"{self.crosstalkShape} {self.nAmp}")
202 # Set default as in __init__
203 self.coeffErr = np.zeros(self.crosstalkShape)
204 self.coeffNum = np.zeros(self.crosstalkShape, dtype=int)
205 self.coeffValid = np.ones(self.crosstalkShape, dtype=bool)
206 self.coeffErrSqr = np.zeros(self.crosstalkShape)
207 self.ampGainRatios = np.zeros(self.crosstalkShape)
208 self.crosstalkRatiosUnits = 'adu'
210 self.interChip = {}
212 self.hasCrosstalk = True
213 self.updateMetadata()
215 return self
217 @classmethod
218 def fromDict(cls, dictionary):
219 """Construct a calibration from a dictionary of properties.
221 Must be implemented by the specific calibration subclasses.
223 Parameters
224 ----------
225 dictionary : `dict`
226 Dictionary of properties.
228 Returns
229 -------
230 calib : `lsst.ip.isr.CalibType`
231 Constructed calibration.
233 Raises
234 ------
235 RuntimeError
236 Raised if the supplied dictionary is for a different
237 calibration.
238 """
239 calib = cls()
241 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
242 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
243 f"found {dictionary['metadata']['OBSTYPE']}")
245 calib.setMetadata(dictionary['metadata'])
247 if 'detectorName' in dictionary:
248 calib._detectorName = dictionary.get('detectorName')
249 elif 'DETECTOR_NAME' in dictionary:
250 calib._detectorName = dictionary.get('DETECTOR_NAME')
251 elif 'DET_NAME' in dictionary['metadata']:
252 calib._detectorName = dictionary['metadata']['DET_NAME']
253 else:
254 calib._detectorName = None
256 if 'detectorSerial' in dictionary:
257 calib._detectorSerial = dictionary.get('detectorSerial')
258 elif 'DETECTOR_SERIAL' in dictionary:
259 calib._detectorSerial = dictionary.get('DETECTOR_SERIAL')
260 elif 'DET_SER' in dictionary['metadata']:
261 calib._detectorSerial = dictionary['metadata']['DET_SER']
262 else:
263 calib._detectorSerial = None
265 if 'detectorId' in dictionary:
266 calib._detectorId = dictionary.get('detectorId')
267 elif 'DETECTOR' in dictionary:
268 calib._detectorId = dictionary.get('DETECTOR')
269 elif 'DETECTOR' in dictionary['metadata']:
270 calib._detectorId = dictionary['metadata']['DETECTOR']
271 elif calib._detectorSerial:
272 calib._detectorId = calib._detectorSerial
273 else:
274 calib._detectorId = None
276 if 'instrument' in dictionary:
277 calib._instrument = dictionary.get('instrument')
278 elif 'INSTRUME' in dictionary['metadata']:
279 calib._instrument = dictionary['metadata']['INSTRUME']
280 else:
281 calib._instrument = None
283 calib.hasCrosstalk = dictionary.get('hasCrosstalk',
284 dictionary['metadata'].get('HAS_CROSSTALK', False))
285 if calib.hasCrosstalk:
286 calib.nAmp = dictionary.get('nAmp', dictionary['metadata'].get('NAMP', 0))
287 calib.crosstalkShape = (calib.nAmp, calib.nAmp)
288 calib.coeffs = np.array(dictionary['coeffs']).reshape(calib.crosstalkShape)
289 calib.crosstalkRatiosUnits = dictionary.get(
290 'crosstalkRatiosUnits',
291 dictionary['metadata'].get('CROSSTALK_RATIOS_UNITS', None))
292 if 'coeffErr' in dictionary:
293 calib.coeffErr = np.array(dictionary['coeffErr']).reshape(calib.crosstalkShape)
294 else:
295 calib.coeffErr = np.zeros_like(calib.coeffs)
296 if 'coeffNum' in dictionary:
297 calib.coeffNum = np.array(dictionary['coeffNum']).reshape(calib.crosstalkShape)
298 else:
299 calib.coeffNum = np.zeros_like(calib.coeffs, dtype=int)
300 if 'coeffValid' in dictionary:
301 calib.coeffValid = np.array(dictionary['coeffValid']).reshape(calib.crosstalkShape)
302 else:
303 calib.coeffValid = np.ones_like(calib.coeffs, dtype=bool)
304 if 'coeffsSqr' in dictionary:
305 calib.coeffsSqr = np.array(dictionary['coeffsSqr']).reshape(calib.crosstalkShape)
306 else:
307 calib.coeffsSqr = np.zeros_like(calib.coeffs)
308 if 'coeffErrSqr' in dictionary:
309 calib.coeffErrSqr = np.array(dictionary['coeffErrSqr']).reshape(calib.crosstalkShape)
310 else:
311 calib.coeffErrSqr = np.zeros_like(calib.coeffs)
312 if 'ampGainRatios' in dictionary:
313 calib.ampGainRatios = np.array(dictionary['ampGainRatios']).reshape(calib.crosstalkShape)
314 else:
315 calib.ampGainRatios = np.zeros_like(calib.coeffs)
316 if 'crosstalkRatiosUnits' in dictionary:
317 calib.crosstalkRatiosUnits = dictionary['crosstalkRatiosUnits']
318 else:
319 calib.crosstalkRatiosUnits = None
321 calib.interChip = dictionary.get('interChip', None)
322 if calib.interChip:
323 for detector in calib.interChip:
324 coeffVector = calib.interChip[detector]
325 calib.interChip[detector] = np.array(coeffVector).reshape(calib.crosstalkShape)
327 calib.updateMetadata()
328 return calib
330 def toDict(self):
331 """Return a dictionary containing the calibration properties.
333 The dictionary should be able to be round-tripped through
334 `fromDict`.
336 Returns
337 -------
338 dictionary : `dict`
339 Dictionary of properties.
340 """
341 self.updateMetadata()
343 outDict = {}
344 metadata = self.getMetadata()
345 outDict['metadata'] = metadata
347 outDict['hasCrosstalk'] = self.hasCrosstalk
348 outDict['nAmp'] = self.nAmp
349 outDict['crosstalkShape'] = self.crosstalkShape
350 outDict['crosstalkRatiosUnits'] = self.crosstalkRatiosUnits
352 ctLength = self.nAmp*self.nAmp
353 outDict['coeffs'] = self.coeffs.reshape(ctLength).tolist()
355 if self.coeffErr is not None:
356 outDict['coeffErr'] = self.coeffErr.reshape(ctLength).tolist()
357 if self.coeffNum is not None:
358 outDict['coeffNum'] = self.coeffNum.reshape(ctLength).tolist()
359 if self.coeffValid is not None:
360 outDict['coeffValid'] = self.coeffValid.reshape(ctLength).tolist()
361 if self.coeffsSqr is not None:
362 outDict['coeffsSqr'] = self.coeffsSqr.reshape(ctLength).tolist()
363 if self.coeffErrSqr is not None:
364 outDict['coeffErrSqr'] = self.coeffErrSqr.reshape(ctLength).tolist()
365 if self.ampGainRatios is not None:
366 outDict['ampGainRatios'] = self.ampGainRatios.reshape(ctLength).tolist()
368 if self.interChip:
369 outDict['interChip'] = dict()
370 for detector in self.interChip:
371 outDict['interChip'][detector] = self.interChip[detector].reshape(ctLength).tolist()
373 return outDict
375 @classmethod
376 def fromTable(cls, tableList):
377 """Construct calibration from a list of tables.
379 This method uses the `fromDict` method to create the
380 calibration, after constructing an appropriate dictionary from
381 the input tables.
383 Parameters
384 ----------
385 tableList : `list` [`lsst.afw.table.Table`]
386 List of tables to use to construct the crosstalk
387 calibration.
389 Returns
390 -------
391 calib : `lsst.ip.isr.CrosstalkCalib`
392 The calibration defined in the tables.
394 """
395 coeffTable = tableList[0]
397 metadata = coeffTable.meta
398 inDict = dict()
399 inDict['metadata'] = metadata
400 inDict['hasCrosstalk'] = metadata['HAS_CROSSTALK']
401 inDict['nAmp'] = metadata['NAMP']
402 calibVersion = metadata['CROSSTALK_VERSION']
403 if calibVersion < 1.1:
404 inDict['crosstalkRatiosUnits'] = ''
405 else:
406 inDict['crosstalkRatiosUnits'] = metadata['CROSSTALK_RATIOS_UNITS']
407 inDict['coeffs'] = coeffTable['CT_COEFFS']
408 if 'CT_ERRORS' in coeffTable.columns:
409 inDict['coeffErr'] = coeffTable['CT_ERRORS']
410 if 'CT_COUNTS' in coeffTable.columns:
411 inDict['coeffNum'] = coeffTable['CT_COUNTS']
412 if 'CT_VALID' in coeffTable.columns:
413 inDict['coeffValid'] = coeffTable['CT_VALID']
414 if 'CT_COEFFS_SQR' in coeffTable.columns:
415 inDict['coeffsSqr'] = coeffTable['CT_COEFFS_SQR']
416 if 'CT_ERRORS_SQR' in coeffTable.columns:
417 inDict['coeffErrSqr'] = coeffTable['CT_ERRORS_SQR']
418 if 'CT_AMP_GAIN_RATIOS' in coeffTable.columns:
419 inDict['ampGainRatios'] = coeffTable['CT_AMP_GAIN_RATIOS']
421 if len(tableList) > 1:
422 inDict['interChip'] = dict()
423 interChipTable = tableList[1]
424 for record in interChipTable:
425 inDict['interChip'][record['IC_SOURCE_DET']] = record['IC_COEFFS']
427 return cls().fromDict(inDict)
429 def toTable(self):
430 """Construct a list of tables containing the information in this
431 calibration.
433 The list of tables should create an identical calibration
434 after being passed to this class's fromTable method.
436 Returns
437 -------
438 tableList : `list` [`lsst.afw.table.Table`]
439 List of tables containing the crosstalk calibration
440 information.
442 """
443 tableList = []
444 self.updateMetadata()
445 catalog = Table([{'CT_COEFFS': self.coeffs.reshape(self.nAmp*self.nAmp),
446 'CT_ERRORS': self.coeffErr.reshape(self.nAmp*self.nAmp),
447 'CT_COUNTS': self.coeffNum.reshape(self.nAmp*self.nAmp),
448 'CT_VALID': self.coeffValid.reshape(self.nAmp*self.nAmp),
449 'CT_COEFFS_SQR': self.coeffsSqr.reshape(self.nAmp*self.nAmp),
450 'CT_ERRORS_SQR': self.coeffErrSqr.reshape(self.nAmp*self.nAmp),
451 'CT_AMP_GAIN_RATIOS': self.ampGainRatios.reshape(self.nAmp*self.nAmp),
452 }])
453 # filter None, because astropy can't deal.
454 inMeta = self.getMetadata().toDict()
455 outMeta = {k: v for k, v in inMeta.items() if v is not None}
456 outMeta.update({k: "" for k, v in inMeta.items() if v is None})
457 catalog.meta = outMeta
458 tableList.append(catalog)
460 if self.interChip:
461 interChipTable = Table([{'IC_SOURCE_DET': sourceDet,
462 'IC_COEFFS': self.interChip[sourceDet].reshape(self.nAmp*self.nAmp)}
463 for sourceDet in self.interChip.keys()])
464 tableList.append(interChipTable)
465 return tableList
467 # Implementation methods.
468 @staticmethod
469 def extractAmp(image, amp, ampTarget, isTrimmed=False):
470 """Extract the image data from an amp, flipped to match ampTarget.
472 Parameters
473 ----------
474 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
475 Image containing the amplifier of interest.
476 amp : `lsst.afw.cameraGeom.Amplifier`
477 Amplifier on image to extract.
478 ampTarget : `lsst.afw.cameraGeom.Amplifier`
479 Target amplifier that the extracted image will be flipped
480 to match.
481 isTrimmed : `bool`
482 The image is already trimmed.
483 TODO : DM-15409 will resolve this.
485 Returns
486 -------
487 output : `lsst.afw.image.Image`
488 Image of the amplifier in the desired configuration.
489 """
490 X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False,
491 lsst.afw.cameraGeom.ReadoutCorner.LR: True,
492 lsst.afw.cameraGeom.ReadoutCorner.UL: False,
493 lsst.afw.cameraGeom.ReadoutCorner.UR: True}
494 Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False,
495 lsst.afw.cameraGeom.ReadoutCorner.LR: False,
496 lsst.afw.cameraGeom.ReadoutCorner.UL: True,
497 lsst.afw.cameraGeom.ReadoutCorner.UR: True}
499 output = image[amp.getBBox() if isTrimmed else amp.getRawDataBBox()]
500 thisAmpCorner = amp.getReadoutCorner()
501 targetAmpCorner = ampTarget.getReadoutCorner()
503 # Flipping is necessary only if the desired configuration doesn't match
504 # what we currently have.
505 xFlip = X_FLIP[targetAmpCorner] ^ X_FLIP[thisAmpCorner]
506 yFlip = Y_FLIP[targetAmpCorner] ^ Y_FLIP[thisAmpCorner]
507 return lsst.afw.math.flipImage(output, xFlip, yFlip)
509 @staticmethod
510 def calculateBackground(mi, badPixels=["BAD"]):
511 """Estimate median background in image.
513 Getting a great background model isn't important for crosstalk
514 correction, since the crosstalk is at a low level. The median should
515 be sufficient.
517 Parameters
518 ----------
519 mi : `lsst.afw.image.MaskedImage`
520 MaskedImage for which to measure background.
521 badPixels : `list` of `str`
522 Mask planes to ignore.
523 Returns
524 -------
525 bg : `float`
526 Median background level.
527 """
528 mask = mi.getMask()
529 stats = lsst.afw.math.StatisticsControl()
530 stats.setAndMask(mask.getPlaneBitMask(badPixels))
531 return lsst.afw.math.makeStatistics(mi, lsst.afw.math.MEDIAN, stats).getValue()
533 def subtractCrosstalk(self, thisExposure, sourceExposure=None, crosstalkCoeffs=None,
534 crosstalkCoeffsSqr=None,
535 badPixels=["BAD"], minPixelToMask=45000,
536 crosstalkStr="CROSSTALK", isTrimmed=False,
537 backgroundMethod="None", doSqrCrosstalk=False):
538 """Subtract the crosstalk from thisExposure, optionally using a
539 different source.
541 We set the mask plane indicated by ``crosstalkStr`` in a target
542 amplifier for pixels in a source amplifier that exceed
543 ``minPixelToMask``. Note that the correction is applied to all pixels
544 in the amplifier, but only those that have a substantial crosstalk
545 are masked with ``crosstalkStr``.
547 The uncorrected image is used as a template for correction. This is
548 good enough if the crosstalk is small (e.g., coefficients < ~ 1e-3),
549 but if it's larger you may want to iterate.
551 Parameters
552 ----------
553 thisExposure : `lsst.afw.image.Exposure`
554 Exposure for which to subtract crosstalk.
555 sourceExposure : `lsst.afw.image.Exposure`, optional
556 Exposure to use as the source of the crosstalk. If not set,
557 thisExposure is used as the source (intra-detector crosstalk).
558 crosstalkCoeffs : `numpy.ndarray`, optional.
559 Coefficients to use to correct crosstalk.
560 crosstalkCoeffsSqr : `numpy.ndarray`, optional.
561 Quadratic coefficients to use to correct crosstalk.
562 badPixels : `list` of `str`, optional
563 Mask planes to ignore.
564 minPixelToMask : `float`, optional
565 Minimum pixel value (relative to the background level) in
566 source amplifier for which to set ``crosstalkStr`` mask plane
567 in target amplifier.
568 crosstalkStr : `str`, optional
569 Mask plane name for pixels greatly modified by crosstalk
570 (above minPixelToMask).
571 isTrimmed : `bool`, optional
572 The image is already trimmed.
573 This should no longer be needed once DM-15409 is resolved.
574 backgroundMethod : `str`, optional
575 Method used to subtract the background. "AMP" uses
576 amplifier-by-amplifier background levels, "DETECTOR" uses full
577 exposure/maskedImage levels. Any other value results in no
578 background subtraction.
579 doSqrCrosstalk: `bool`, optional
580 Should the quadratic crosstalk coefficients be used for the
581 crosstalk correction?
583 Notes
584 -----
586 For a given image I, we want to find the crosstalk subtrahend
587 image CT, such that
588 I_corrected = I - CT
589 The subtrahend image is the sum of all crosstalk contributions
590 that appear in I, so we can build it up by amplifier. Each
591 amplifier A in image I sees the contributions from all other
592 amplifiers B_v != A. For the current linear model, we set `sImage`
593 equal to the segment of the subtrahend image CT corresponding to
594 amplifier A, and then build it up as:
595 simage_linear = sum_v coeffsA_v * (B_v - bkg_v) where coeffsA_v
596 is the vector of crosstalk coefficients for sources that cause
597 images in amplifier A. The bkg_v term in this equation is
598 identically 0.0 for all cameras except obs_subaru (and is only
599 non-zero there for historical reasons).
600 To include the non-linear term, we can again add to the subtrahend
601 image using the same loop, as:
603 simage_nonlinear = sum_v (coeffsA_v * B_v) + (NLcoeffsA_v * B_v * B_v)
604 = sum_v linear_term_v + nonlinear_term_v
606 where coeffsA_v is the linear term, and NLcoeffsA_v are the quadratic
607 component. For LSSTCam, it has been observed that the linear_term_v >>
608 nonlinear_term_v.
609 """
610 mi = thisExposure.getMaskedImage()
611 mask = mi.getMask()
612 detector = thisExposure.getDetector()
613 if self.hasCrosstalk is False:
614 self.fromDetector(detector, coeffVector=crosstalkCoeffs)
616 numAmps = len(detector)
617 if numAmps != self.nAmp:
618 raise RuntimeError(f"Crosstalk built for {self.nAmp} in {self._detectorName}, received "
619 f"{numAmps} in {detector.getName()}")
621 if doSqrCrosstalk and crosstalkCoeffsSqr is None:
622 raise RuntimeError("Attempted to perform NL crosstalk correction without NL "
623 "crosstalk coefficients.")
625 if sourceExposure:
626 source = sourceExposure.getMaskedImage()
627 sourceDetector = sourceExposure.getDetector()
628 else:
629 source = mi
630 sourceDetector = detector
632 if crosstalkCoeffs is not None:
633 coeffs = crosstalkCoeffs
634 else:
635 coeffs = self.coeffs
636 self.log.debug("CT COEFF: %s", coeffs)
638 if doSqrCrosstalk:
639 if crosstalkCoeffsSqr is not None:
640 coeffsSqr = crosstalkCoeffsSqr
641 else:
642 coeffsSqr = self.coeffsSqr
643 self.log.debug("CT COEFF SQR: %s", coeffsSqr)
644 # Set background level based on the requested method. The
645 # thresholdBackground holds the offset needed so that we only mask
646 # pixels high relative to the background, not in an absolute
647 # sense.
648 thresholdBackground = self.calculateBackground(source, badPixels)
650 backgrounds = [0.0 for amp in sourceDetector]
651 if backgroundMethod is None:
652 pass
653 elif backgroundMethod == "AMP":
654 backgrounds = [self.calculateBackground(source[amp.getBBox()], badPixels)
655 for amp in sourceDetector]
656 elif backgroundMethod == "DETECTOR":
657 backgrounds = [self.calculateBackground(source, badPixels) for amp in sourceDetector]
659 # Set the crosstalkStr bit for the bright pixels (those which will have
660 # significant crosstalk correction)
661 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
662 footprints = lsst.afw.detection.FootprintSet(source,
663 lsst.afw.detection.Threshold(minPixelToMask
664 + thresholdBackground))
665 footprints.setMask(mask, crosstalkStr)
666 crosstalk = mask.getPlaneBitMask(crosstalkStr)
668 # Define a subtrahend image to contain all the scaled crosstalk signals
669 subtrahend = source.Factory(source.getBBox())
670 subtrahend.set((0, 0, 0))
672 coeffs = coeffs.transpose()
673 # Apply NL coefficients
674 if doSqrCrosstalk:
675 coeffsSqr = coeffsSqr.transpose()
676 mi2 = mi.clone()
677 mi2.scaledMultiplies(1.0, mi)
678 for ss, sAmp in enumerate(sourceDetector):
679 sImage = subtrahend[sAmp.getBBox() if isTrimmed else sAmp.getRawDataBBox()]
680 for tt, tAmp in enumerate(detector):
681 if coeffs[ss, tt] == 0.0:
682 continue
683 tImage = self.extractAmp(mi, tAmp, sAmp, isTrimmed)
684 tImage.getMask().getArray()[:] &= crosstalk # Remove all other masks
685 tImage -= backgrounds[tt]
686 sImage.scaledPlus(coeffs[ss, tt], tImage)
687 # Add the nonlinear term
688 if doSqrCrosstalk:
689 tImageSqr = self.extractAmp(mi2, tAmp, sAmp, isTrimmed)
690 sImage.scaledPlus(coeffsSqr[ss, tt], tImageSqr)
692 # Set crosstalkStr bit only for those pixels that have been
693 # significantly modified (i.e., those masked as such in 'subtrahend'),
694 # not necessarily those that are bright originally.
695 mask.clearMaskPlane(crosstalkPlane)
696 mi -= subtrahend # also sets crosstalkStr bit for bright pixels
698 def subtractCrosstalkParallelOverscanRegion(self, thisExposure, crosstalkCoeffs=None,
699 crosstalkCoeffsSqr=None,
700 badPixels=["BAD"], crosstalkStr="CROSSTALK",
701 detectorConfig=None, doSqrCrosstalk=False):
702 """Subtract crosstalk just from the parallel overscan region.
704 This assumes that serial overscan has been previously subtracted.
706 Parameters
707 ----------
708 thisExposure : `lsst.afw.image.Exposure`
709 Exposure for which to subtract crosstalk.
710 crosstalkCoeffs : `numpy.ndarray`, optional.
711 Coefficients to use to correct crosstalk.
712 crosstalkCoeffsSqr : `numpy.ndarray`, optional.
713 Quadratic coefficients to use to correct crosstalk.
714 badPixels : `list` of `str`, optional
715 Mask planes to ignore.
716 crosstalkStr : `str`, optional
717 Mask plane name for pixels greatly modified by crosstalk
718 (above minPixelToMask).
719 detectorConfig : `lsst.ip.isr.overscanDetectorConfig`, optional
720 Per-amplifier configs to use.
721 doSqrCrosstalk: `bool`, optional
722 Should the quadratic crosstalk coefficients be used for the
723 crosstalk correction?
724 """
725 mi = thisExposure.getMaskedImage()
726 mask = mi.getMask()
727 detector = thisExposure.getDetector()
728 if self.hasCrosstalk is False:
729 self.fromDetector(detector, coeffVector=crosstalkCoeffs)
731 numAmps = len(detector)
732 if numAmps != self.nAmp:
733 raise RuntimeError(f"Crosstalk built for {self.nAmp} in {self._detectorName}, received "
734 f"{numAmps} in {detector.getName()}")
736 if doSqrCrosstalk and crosstalkCoeffsSqr is None:
737 raise RuntimeError("Attempted to perform NL crosstalk correction without NL "
738 "crosstalk coefficients.")
740 source = mi
741 sourceDetector = detector
743 if crosstalkCoeffs is not None:
744 coeffs = crosstalkCoeffs
745 else:
746 coeffs = self.coeffs
747 if doSqrCrosstalk:
748 if crosstalkCoeffsSqr is not None:
749 coeffsSqr = crosstalkCoeffsSqr
750 else:
751 coeffsSqr = self.coeffsSqr
752 self.log.debug("CT COEFF SQR: %s", coeffsSqr)
754 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
755 crosstalk = mask.getPlaneBitMask(crosstalkStr)
757 subtrahend = source.Factory(source.getBBox())
758 subtrahend.set((0, 0, 0))
760 coeffs = coeffs.transpose()
761 # Apply NL coefficients
762 if doSqrCrosstalk:
763 coeffsSqr = coeffsSqr.transpose()
764 mi2 = mi.clone()
765 mi2.scaledMultiplies(1.0, mi)
766 for ss, sAmp in enumerate(sourceDetector):
767 if detectorConfig is not None:
768 ampConfig = detectorConfig.getOverscanAmpconfig(sAmp.getName())
769 if not ampConfig.doParallelOverscanCrosstalk:
770 # Skip crosstalk correction for this amplifier.
771 continue
773 sImage = subtrahend[sAmp.getRawParallelOverscanBBox()]
774 for tt, tAmp in enumerate(detector):
775 if coeffs[ss, tt] == 0.0:
776 continue
777 tImage = self.extractAmp(mi, tAmp, sAmp, False, parallelOverscan=True)
778 tImage.getMask().getArray()[:] &= crosstalk # Remove all other masks
779 sImage.scaledPlus(coeffs[ss, tt], tImage)
780 # Add the nonlinear term, if any.
781 if doSqrCrosstalk:
782 tImageSqr = self.extractAmp(mi2, tAmp, sAmp, False, parallelOverscan=True)
783 sImage.scaledPlus(coeffsSqr[ss, tt], tImageSqr)
784 # Set crosstalkStr bit only for those pixels that have been
785 # significantly modified (i.e., those masked as such in 'subtrahend'),
786 # not necessarily those that are bright originally.
787 mask.clearMaskPlane(crosstalkPlane)
788 mi -= subtrahend # also sets crosstalkStr bit for bright pixels
791class CrosstalkConfig(Config):
792 """Configuration for intra-detector crosstalk removal."""
793 minPixelToMask = Field(
794 dtype=float,
795 doc="Set crosstalk mask plane for pixels over this value.",
796 default=45000
797 )
798 crosstalkMaskPlane = Field(
799 dtype=str,
800 doc="Name for crosstalk mask plane.",
801 default="CROSSTALK"
802 )
803 crosstalkBackgroundMethod = ChoiceField(
804 dtype=str,
805 doc="Type of background subtraction to use when applying correction.",
806 default="None",
807 allowed={
808 "None": "Do no background subtraction.",
809 "AMP": "Subtract amplifier-by-amplifier background levels.",
810 "DETECTOR": "Subtract detector level background."
811 },
812 )
813 useConfigCoefficients = Field(
814 dtype=bool,
815 doc="Ignore the detector crosstalk information in favor of CrosstalkConfig values?",
816 default=False,
817 )
818 crosstalkValues = ListField(
819 dtype=float,
820 doc=("Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 "
821 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. "
822 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column "
823 "vector [corr0 corr1 corr2 ...]^T."),
824 default=[0.0],
825 )
826 crosstalkShape = ListField(
827 dtype=int,
828 doc="Shape of the coefficient array. This should be equal to [nAmp, nAmp].",
829 default=[1],
830 )
831 doQuadraticCrosstalkCorrection = Field(
832 dtype=bool,
833 doc="Use quadratic crosstalk coefficients in the crosstalk correction",
834 default=False,
835 )
837 def getCrosstalk(self, detector=None):
838 """Return a 2-D numpy array of crosstalk coefficients in the proper
839 shape.
841 Parameters
842 ----------
843 detector : `lsst.afw.cameraGeom.detector`
844 Detector that is to be crosstalk corrected.
846 Returns
847 -------
848 coeffs : `numpy.ndarray`
849 Crosstalk coefficients that can be used to correct the detector.
851 Raises
852 ------
853 RuntimeError
854 Raised if no coefficients could be generated from this
855 detector/configuration.
856 """
857 if self.useConfigCoefficients is True:
858 coeffs = np.array(self.crosstalkValues).reshape(self.crosstalkShape)
859 if detector is not None:
860 nAmp = len(detector)
861 if coeffs.shape != (nAmp, nAmp):
862 raise RuntimeError("Constructed crosstalk coeffients do not match detector shape. "
863 f"{coeffs.shape} {nAmp}")
864 return coeffs
865 elif detector is not None and detector.hasCrosstalk() is True:
866 # Assume the detector defines itself consistently.
867 return detector.getCrosstalk()
868 else:
869 raise RuntimeError("Attempted to correct crosstalk without crosstalk coefficients")
871 def hasCrosstalk(self, detector=None):
872 """Return a boolean indicating if crosstalk coefficients exist.
874 Parameters
875 ----------
876 detector : `lsst.afw.cameraGeom.detector`
877 Detector that is to be crosstalk corrected.
879 Returns
880 -------
881 hasCrosstalk : `bool`
882 True if this detector/configuration has crosstalk coefficients
883 defined.
884 """
885 if self.useConfigCoefficients is True and self.crosstalkValues is not None:
886 return True
887 elif detector is not None and detector.hasCrosstalk() is True:
888 return True
889 else:
890 return False
893class CrosstalkTask(Task):
894 """Apply intra-detector crosstalk correction."""
895 ConfigClass = CrosstalkConfig
896 _DefaultName = 'isrCrosstalk'
898 def run(self,
899 exposure, crosstalk=None,
900 crosstalkSources=None, isTrimmed=False, camera=None, parallelOverscanRegion=False,
901 detectorConfig=None,
902 ):
903 """Apply intra-detector crosstalk correction
905 Parameters
906 ----------
907 exposure : `lsst.afw.image.Exposure`
908 Exposure for which to remove crosstalk.
909 crosstalkCalib : `lsst.ip.isr.CrosstalkCalib`, optional
910 External crosstalk calibration to apply. Constructed from
911 detector if not found.
912 crosstalkSources : `defaultdict`, optional
913 Image data for other detectors that are sources of
914 crosstalk in exposure. The keys are expected to be names
915 of the other detectors, with the values containing
916 `lsst.afw.image.Exposure` at the same level of processing
917 as ``exposure``.
918 The default for intra-detector crosstalk here is None.
919 isTrimmed : `bool`, optional
920 The image is already trimmed.
921 This should no longer be needed once DM-15409 is resolved.
922 camera : `lsst.afw.cameraGeom.Camera`, optional
923 Camera associated with this exposure. Only used for
924 inter-chip matching.
925 parallelOverscanRegion : `bool`, optional
926 Do subtraction in parallel overscan region (only)?
927 detectorConfig : `lsst.ip.isr.OverscanDetectorConfig`, optional
928 Per-amplifier configs used when parallelOverscanRegion=True.
930 Raises
931 ------
932 RuntimeError
933 Raised if called for a detector that does not have a
934 crosstalk correction. Also raised if the crosstalkSource
935 is not an expected type.
936 """
937 if not crosstalk:
938 crosstalk = CrosstalkCalib(log=self.log)
939 crosstalk = crosstalk.fromDetector(exposure.getDetector(),
940 coeffVector=self.config.crosstalkValues)
941 if not crosstalk.log:
942 crosstalk.log = self.log
944 doSqrCrosstalk = self.config.doQuadraticCrosstalkCorrection
945 if doSqrCrosstalk and crosstalk.coeffsSqr is None:
946 raise RuntimeError("Attempted to perform NL crosstalk correction without NL "
947 "crosstalk coefficients.")
948 if doSqrCrosstalk:
949 crosstalkCoeffsSqr = crosstalk.coeffsSqr
950 else:
951 crosstalkCoeffsSqr = None
953 if not crosstalk.hasCrosstalk:
954 raise RuntimeError("Attempted to correct crosstalk without crosstalk coefficients.")
955 elif parallelOverscanRegion:
956 self.log.info("Applying crosstalk correction to parallel overscan region.")
957 crosstalk.subtractCrosstalkParallelOverscanRegion(
958 exposure,
959 crosstalkCoeffs=crosstalk.coeffs,
960 crosstalkCoeffsSqr=crosstalkCoeffsSqr,
961 detectorConfig=detectorConfig,
962 doSqrCrosstalk=doSqrCrosstalk,
963 )
964 else:
965 self.log.info("Applying crosstalk correction.")
966 crosstalk.subtractCrosstalk(exposure, crosstalkCoeffs=crosstalk.coeffs,
967 crosstalkCoeffsSqr=crosstalkCoeffsSqr,
968 minPixelToMask=self.config.minPixelToMask,
969 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed,
970 backgroundMethod=self.config.crosstalkBackgroundMethod,
971 doSqrCrosstalk=doSqrCrosstalk)
973 if crosstalk.interChip:
974 if crosstalkSources:
975 # Parse crosstalkSources: Identify which detectors we have
976 # available
977 if isinstance(crosstalkSources[0], lsst.afw.image.Exposure):
978 # Received afwImage.Exposure
979 sourceNames = [exp.getDetector().getName() for exp in crosstalkSources]
980 elif isinstance(crosstalkSources[0], lsst.daf.butler.DeferredDatasetHandle):
981 # Received dafButler.DeferredDatasetHandle
982 detectorList = [source.dataId['detector'] for source in crosstalkSources]
983 sourceNames = [camera[detector].getName() for detector in detectorList]
984 else:
985 raise RuntimeError("Unknown object passed as crosstalk sources.",
986 type(crosstalkSources[0]))
988 for detName in crosstalk.interChip:
989 if detName not in sourceNames:
990 self.log.warning("Crosstalk lists %s, not found in sources: %s",
991 detName, sourceNames)
992 continue
993 # Get the coefficients.
994 interChipCoeffs = crosstalk.interChip[detName]
996 sourceExposure = crosstalkSources[sourceNames.index(detName)]
997 if isinstance(sourceExposure, lsst.daf.butler.DeferredDatasetHandle):
998 # Dereference the dafButler.DeferredDatasetHandle.
999 sourceExposure = sourceExposure.get()
1000 if not isinstance(sourceExposure, lsst.afw.image.Exposure):
1001 raise RuntimeError("Unknown object passed as crosstalk sources.",
1002 type(sourceExposure))
1004 self.log.info("Correcting detector %s with ctSource %s",
1005 exposure.getDetector().getName(),
1006 sourceExposure.getDetector().getName())
1007 crosstalk.subtractCrosstalk(exposure, sourceExposure=sourceExposure,
1008 crosstalkCoeffs=interChipCoeffs,
1009 minPixelToMask=self.config.minPixelToMask,
1010 crosstalkStr=self.config.crosstalkMaskPlane,
1011 isTrimmed=isTrimmed,
1012 backgroundMethod=self.config.crosstalkBackgroundMethod)
1013 else:
1014 self.log.warning("Crosstalk contains interChip coefficients, but no sources found!")
1017class NullCrosstalkTask(CrosstalkTask):
1018 def run(self, exposure, crosstalkSources=None):
1019 self.log.info("Not performing any crosstalk correction")