Coverage for python/lsst/ip/isr/crosstalk.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Apply intra-detector crosstalk corrections
24"""
25import numpy as np
27import lsst.afw.math
28import lsst.afw.table
29import lsst.afw.detection
30from lsst.pex.config import Config, Field, ChoiceField, ListField
31from lsst.pipe.base import Task
33__all__ = ["CrosstalkConfig", "CrosstalkTask", "subtractCrosstalk", "writeCrosstalkCoeffs",
34 "NullCrosstalkTask"]
37class CrosstalkConfig(Config):
38 """Configuration for intra-detector crosstalk removal."""
39 minPixelToMask = Field(
40 dtype=float,
41 doc="Set crosstalk mask plane for pixels over this value.",
42 default=45000
43 )
44 crosstalkMaskPlane = Field(
45 dtype=str,
46 doc="Name for crosstalk mask plane.",
47 default="CROSSTALK"
48 )
49 crosstalkBackgroundMethod = ChoiceField(
50 dtype=str,
51 doc="Type of background subtraction to use when applying correction.",
52 default="None",
53 allowed={
54 "None": "Do no background subtraction.",
55 "AMP": "Subtract amplifier-by-amplifier background levels.",
56 "DETECTOR": "Subtract detector level background."
57 },
58 )
59 useConfigCoefficients = Field(
60 dtype=bool,
61 doc="Ignore the detector crosstalk information in favor of CrosstalkConfig values?",
62 default=False,
63 )
64 crosstalkValues = ListField(
65 dtype=float,
66 doc=("Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 "
67 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. "
68 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column "
69 "vector [corr0 corr1 corr2 ...]^T."),
70 default=[0.0],
71 )
72 crosstalkShape = ListField(
73 dtype=int,
74 doc="Shape of the coefficient array. This should be equal to [nAmp, nAmp].",
75 default=[1],
76 )
78 def getCrosstalk(self, detector=None):
79 """Return a 2-D numpy array of crosstalk coefficients in the proper shape.
81 Parameters
82 ----------
83 detector : `lsst.afw.cameraGeom.detector`
84 Detector that is to be crosstalk corrected.
86 Returns
87 -------
88 coeffs : `numpy.ndarray`
89 Crosstalk coefficients that can be used to correct the detector.
91 Raises
92 ------
93 RuntimeError
94 Raised if no coefficients could be generated from this detector/configuration.
95 """
96 if self.useConfigCoefficients is True:
97 coeffs = np.array(self.crosstalkValues).reshape(self.crosstalkShape)
98 if detector is not None:
99 nAmp = len(detector)
100 if coeffs.shape != (nAmp, nAmp):
101 raise RuntimeError("Constructed crosstalk coeffients do not match detector shape. " +
102 f"{coeffs.shape} {nAmp}")
103 return coeffs
104 elif detector is not None and detector.hasCrosstalk() is True:
105 # Assume the detector defines itself consistently.
106 return detector.getCrosstalk()
107 else:
108 raise RuntimeError("Attempted to correct crosstalk without crosstalk coefficients")
110 def hasCrosstalk(self, detector=None):
111 """Return a boolean indicating if crosstalk coefficients exist.
113 Parameters
114 ----------
115 detector : `lsst.afw.cameraGeom.detector`
116 Detector that is to be crosstalk corrected.
118 Returns
119 -------
120 hasCrosstalk : `bool`
121 True if this detector/configuration has crosstalk coefficients defined.
122 """
123 if self.useConfigCoefficients is True and self.crosstalkValues is not None:
124 return True
125 elif detector is not None and detector.hasCrosstalk() is True:
126 return True
127 else:
128 return False
131class CrosstalkTask(Task):
132 """Apply intra-detector crosstalk correction."""
133 ConfigClass = CrosstalkConfig
134 _DefaultName = 'isrCrosstalk'
136 def prepCrosstalk(self, dataRef):
137 """Placeholder for crosstalk preparation method, e.g., for inter-detector crosstalk.
139 Parameters
140 ----------
141 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
142 Butler reference of the detector data to be processed.
144 See also
145 --------
146 lsst.obs.decam.crosstalk.DecamCrosstalkTask.prepCrosstalk
147 """
148 return
150 def run(self, exposure, crosstalkSources=None, isTrimmed=False):
151 """Apply intra-detector crosstalk correction
153 Parameters
154 ----------
155 exposure : `lsst.afw.image.Exposure`
156 Exposure for which to remove crosstalk.
157 crosstalkSources : `defaultdict`, optional
158 Image data and crosstalk coefficients from other detectors/amps that are
159 sources of crosstalk in exposure.
160 The default for intra-detector crosstalk here is None.
161 isTrimmed : `bool`
162 The image is already trimmed.
163 This should no longer be needed once DM-15409 is resolved.
165 Raises
166 ------
167 RuntimeError
168 Raised if called for a detector that does not have a
169 crosstalk correction.
170 TypeError
171 Raised if crosstalkSources is not None
172 and not a numpy array or a dictionary.
173 """
174 if crosstalkSources is not None:
175 if isinstance(crosstalkSources, np.ndarray):
176 coeffs = crosstalkSources
177 elif isinstance(crosstalkSources, dict):
178 # Nested dictionary produced by `measureCrosstalk.py`
179 # There are two keys first in the nested dictionary
180 for fKey, fValue in crosstalkSources.items():
181 for sKey, sValue in fValue.items():
182 firstKey = fKey
183 secondKey = sKey
184 tempDict = crosstalkSources[firstKey][secondKey]
185 coeffs = []
186 for thirdKey in tempDict:
187 tempList = []
188 for fourthKey in tempDict[thirdKey]:
189 value = tempDict[thirdKey][fourthKey]
190 tempList.append(value)
191 coeffs.append(tempList)
192 coeffs = np.array(coeffs)
193 else:
194 raise TypeError("crosstalkSources not of the correct type: `np.array` or `dict`")
195 else:
196 detector = exposure.getDetector()
197 if not self.config.hasCrosstalk(detector=detector):
198 raise RuntimeError("Attempted to correct crosstalk without crosstalk coefficients")
199 coeffs = self.config.getCrosstalk(detector=detector)
201 self.log.info("Applying crosstalk correction.")
202 subtractCrosstalk(exposure, crosstalkCoeffs=coeffs,
203 minPixelToMask=self.config.minPixelToMask,
204 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed,
205 backgroundMethod=self.config.crosstalkBackgroundMethod)
208# Flips required to get the corner to the lower-left
209# (an arbitrary choice; flips are relative, so the choice of reference here is not important)
210X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False, lsst.afw.cameraGeom.ReadoutCorner.LR: True,
211 lsst.afw.cameraGeom.ReadoutCorner.UL: False, lsst.afw.cameraGeom.ReadoutCorner.UR: True}
212Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False, lsst.afw.cameraGeom.ReadoutCorner.LR: False,
213 lsst.afw.cameraGeom.ReadoutCorner.UL: True, lsst.afw.cameraGeom.ReadoutCorner.UR: True}
216class NullCrosstalkTask(CrosstalkTask):
217 def run(self, exposure, crosstalkSources=None):
218 self.log.info("Not performing any crosstalk correction")
221def extractAmp(image, amp, corner, isTrimmed=False):
222 """Return an image of the amp
224 The returned image will have the amp's readout corner in the
225 nominated `corner`.
227 Parameters
228 ----------
229 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
230 Image containing the amplifier of interest.
231 amp : `lsst.afw.table.AmpInfoRecord`
232 Amplifier information.
233 corner : `lsst.afw.table.ReadoutCorner` or `None`
234 Corner in which to put the amp's readout corner, or `None` for
235 no flipping.
236 isTrimmed : `bool`
237 The image is already trimmed.
238 This should no longer be needed once DM-15409 is resolved.
240 Returns
241 -------
242 output : `lsst.afw.image.Image`
243 Image of the amplifier in the standard configuration.
244 """
245 output = image[amp.getBBox() if isTrimmed else amp.getRawDataBBox()]
246 ampCorner = amp.getReadoutCorner()
247 # Flipping is necessary only if the desired configuration doesn't match what we currently have
248 xFlip = X_FLIP[corner] ^ X_FLIP[ampCorner]
249 yFlip = Y_FLIP[corner] ^ Y_FLIP[ampCorner]
250 return lsst.afw.math.flipImage(output, xFlip, yFlip)
253def calculateBackground(mi, badPixels=["BAD"]):
254 """Calculate median background in image
256 Getting a great background model isn't important for crosstalk correction,
257 since the crosstalk is at a low level. The median should be sufficient.
259 Parameters
260 ----------
261 mi : `lsst.afw.image.MaskedImage`
262 MaskedImage for which to measure background.
263 badPixels : `list` of `str`
264 Mask planes to ignore.
266 Returns
267 -------
268 bg : `float`
269 Median background level.
270 """
271 mask = mi.getMask()
272 stats = lsst.afw.math.StatisticsControl()
273 stats.setAndMask(mask.getPlaneBitMask(badPixels))
274 return lsst.afw.math.makeStatistics(mi, lsst.afw.math.MEDIAN, stats).getValue()
277def subtractCrosstalk(exposure, crosstalkCoeffs=None,
278 badPixels=["BAD"], minPixelToMask=45000,
279 crosstalkStr="CROSSTALK", isTrimmed=False,
280 backgroundMethod="None"):
281 """Subtract the intra-detector crosstalk from an exposure
283 We set the mask plane indicated by ``crosstalkStr`` in a target amplifier
284 for pixels in a source amplifier that exceed `minPixelToMask`. Note that
285 the correction is applied to all pixels in the amplifier, but only those
286 that have a substantial crosstalk are masked with ``crosstalkStr``.
288 The uncorrected image is used as a template for correction. This is good
289 enough if the crosstalk is small (e.g., coefficients < ~ 1e-3), but if it's
290 larger you may want to iterate.
292 This method needs unittests (DM-18876), but such testing requires
293 DM-18610 to allow the test detector to have the crosstalk
294 parameters set.
296 Parameters
297 ----------
298 exposure : `lsst.afw.image.Exposure`
299 Exposure for which to subtract crosstalk.
300 crosstalkCoeffs : `numpy.ndarray`
301 Coefficients to use to correct crosstalk.
302 badPixels : `list` of `str`
303 Mask planes to ignore.
304 minPixelToMask : `float`
305 Minimum pixel value (relative to the background level) in
306 source amplifier for which to set ``crosstalkStr`` mask plane
307 in target amplifier.
308 crosstalkStr : `str`
309 Mask plane name for pixels greatly modified by crosstalk.
310 isTrimmed : `bool`
311 The image is already trimmed.
312 This should no longer be needed once DM-15409 is resolved.
313 backgroundMethod : `str`
314 Method used to subtract the background. "AMP" uses
315 amplifier-by-amplifier background levels, "DETECTOR" uses full
316 exposure/maskedImage levels. Any other value results in no
317 background subtraction.
318 """
319 mi = exposure.getMaskedImage()
320 mask = mi.getMask()
322 ccd = exposure.getDetector()
323 numAmps = len(ccd)
324 if crosstalkCoeffs is None:
325 coeffs = ccd.getCrosstalk()
326 else:
327 coeffs = crosstalkCoeffs
328 assert coeffs.shape == (numAmps, numAmps)
330 # Set background level based on the requested method. The
331 # thresholdBackground holds the offset needed so that we only mask
332 # pixels high relative to the background, not in an absolute
333 # sense.
334 thresholdBackground = calculateBackground(mi, badPixels)
336 backgrounds = [0.0 for amp in ccd]
337 if backgroundMethod is None:
338 pass
339 elif backgroundMethod == "AMP":
340 backgrounds = [calculateBackground(mi[amp.getBBox()], badPixels) for amp in ccd]
341 elif backgroundMethod == "DETECTOR":
342 backgrounds = [calculateBackground(mi, badPixels) for amp in ccd]
344 # Set the crosstalkStr bit for the bright pixels (those which will have significant crosstalk correction)
345 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
346 footprints = lsst.afw.detection.FootprintSet(mi, lsst.afw.detection.Threshold(minPixelToMask +
347 thresholdBackground))
348 footprints.setMask(mask, crosstalkStr)
349 crosstalk = mask.getPlaneBitMask(crosstalkStr)
351 # Do pixel level crosstalk correction.
352 subtrahend = mi.Factory(mi.getBBox())
353 subtrahend.set((0, 0, 0))
354 for ii, iAmp in enumerate(ccd):
355 iImage = subtrahend[iAmp.getBBox() if isTrimmed else iAmp.getRawDataBBox()]
356 for jj, jAmp in enumerate(ccd):
357 if ii == jj:
358 assert coeffs[ii, jj] == 0.0
359 if coeffs[ii, jj] == 0.0:
360 continue
362 jImage = extractAmp(mi, jAmp, iAmp.getReadoutCorner(), isTrimmed)
363 jImage.getMask().getArray()[:] &= crosstalk # Remove all other masks
364 jImage -= backgrounds[jj]
366 iImage.scaledPlus(coeffs[ii, jj], jImage)
368 # Set crosstalkStr bit only for those pixels that have been significantly modified (i.e., those
369 # masked as such in 'subtrahend'), not necessarily those that are bright originally.
370 mask.clearMaskPlane(crosstalkPlane)
371 mi -= subtrahend # also sets crosstalkStr bit for bright pixels
374def writeCrosstalkCoeffs(outputFileName, coeff, det=None, crosstalkName="Unknown", indent=2):
375 """Write a yaml file containing the crosstalk coefficients
377 The coeff array is indexed by [i, j] where i and j are amplifiers
378 corresponding to the amplifiers in det
380 Parameters
381 ----------
382 outputFileName : `str`
383 Name of output yaml file
384 coeff : `numpy.array(namp, namp)`
385 numpy array of coefficients
386 det : `lsst.afw.cameraGeom.Detector`
387 Used to provide the list of amplifier names;
388 if None use ['0', '1', ...]
389 ccdType : `str`
390 Name of detector, used to index the yaml file
391 If all detectors are identical could be the type (e.g. ITL)
392 indent : `int`
393 Indent width to use when writing the yaml file
394 """
396 if det is None:
397 ampNames = [str(i) for i in range(coeff.shape[0])]
398 else:
399 ampNames = [a.getName() for a in det]
401 assert coeff.shape == (len(ampNames), len(ampNames))
403 dIndent = indent
404 indent = 0
405 with open(outputFileName, "w") as fd:
406 print(indent*" " + "crosstalk :", file=fd)
407 indent += dIndent
408 print(indent*" " + "%s :" % crosstalkName, file=fd)
409 indent += dIndent
411 for i, ampNameI in enumerate(ampNames):
412 print(indent*" " + "%s : {" % ampNameI, file=fd)
413 indent += dIndent
414 print(indent*" ", file=fd, end='')
416 for j, ampNameJ in enumerate(ampNames):
417 print("%s : %11.4e, " % (ampNameJ, coeff[i, j]), file=fd,
418 end='\n' + indent*" " if j%4 == 3 else '')
419 print("}", file=fd)
421 indent -= dIndent