23 Measure intra-detector crosstalk coefficients.
26 __all__ = [
"MeasureCrosstalkConfig",
"MeasureCrosstalkTask"]
32 from lsstDebug
import getDebugFrame
36 from lsst.pex.config
import Config, Field, ListField, ConfigurableField
39 from .crosstalk
import CrosstalkCalib
40 from .calibType
import IsrProvenance
41 from .isrTask
import IsrTask
45 """Configuration for MeasureCrosstalkTask."""
46 isr = ConfigurableField(
48 doc=
"Instrument signature removal task to use to process data."
53 doc=
"Minimum level of source pixels for which to measure crosstalk."
58 doc=
"Rerun the ISR, even if postISRCCD files are available?"
62 default=[
"SAT",
"BAD",
"INTRP"],
63 doc=
"Mask planes to ignore when identifying source pixels."
68 doc=
"Number of rejection iterations for final coefficient calculation."
73 doc=
"Rejection threshold (sigma) for final coefficient calculation."
78 doc=
"Have the amplifiers been trimmed before measuring CT?"
82 Config.setDefaults(self)
85 self.
isr.doWrite =
False
86 self.
isr.doOverscan =
True
87 self.
isr.doAssembleCcd =
True
88 self.
isr.doBias =
True
89 self.
isr.doVariance =
False
90 self.
isr.doLinearize =
True
91 self.
isr.doCrosstalk =
False
92 self.
isr.doBrighterFatter =
False
93 self.
isr.doDark =
False
94 self.
isr.doStrayLight =
False
95 self.
isr.doFlat =
False
96 self.
isr.doFringe =
False
97 self.
isr.doApplyGains =
False
98 self.
isr.doDefect =
True
99 self.
isr.doSaturationInterpolation =
False
100 self.
isr.growSaturationFootprintSize = 0
104 """Measure intra-detector crosstalk.
108 The crosstalk this method measures assumes that when a bright
109 pixel is found in one detector amplifier, all other detector
110 amplifiers may see an increase in the same pixel location
111 (relative to the readout amplifier) as these other pixels are read
112 out at the same time.
114 After processing each input exposure through a limited set of ISR
115 stages, bright unmasked pixels above the threshold are identified.
116 The potential CT signal is found by taking the ratio of the
117 appropriate background-subtracted pixel value on the other
118 amplifiers to the input value on the source amplifier. If the
119 source amplifier has a large number of bright pixels as well, the
120 background level may be elevated, leading to poor ratio
123 The set of ratios found between each pair of amplifiers across all
124 input exposures is then gathered to produce the final CT
125 coefficients. The sigma-clipped mean and sigma are returned from
126 these sets of ratios, with the coefficient to supply to the ISR
127 CrosstalkTask() being the multiplicative inverse of these values.
129 ConfigClass = MeasureCrosstalkConfig
130 _DefaultName =
"measureCrosstalk"
133 CmdLineTask.__init__(self, *args, **kwargs)
134 self.makeSubtask(
"isr")
138 def _makeArgumentParser(cls):
139 parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
140 parser.add_argument(
"--crosstalkName",
141 help=
"Name for this set of crosstalk coefficients", default=
"Unknown")
142 parser.add_argument(
"--outputFileName",
143 help=
"Name of yaml file to which to write crosstalk coefficients")
144 parser.add_argument(
"--dump-ratios", dest=
"dumpRatios",
145 help=
"Name of pickle file to which to write crosstalk ratios")
150 """Collate crosstalk results from multiple exposures.
152 Process all input exposures through runDataRef, construct
153 final measurements from the final list of results from each
154 input, and persist the output calibration.
156 This method will be deprecated as part of DM-24760.
160 coeff : `numpy.ndarray`
161 Crosstalk coefficients.
162 coeffErr : `numpy.ndarray`
163 Crosstalk coefficient errors.
164 coeffNum : `numpy.ndarray`
165 Number of pixels used for crosstalk measurement.
166 calib : `lsst.ip.isr.CrosstalkCalib`
167 Crosstalk object created from the measurements.
170 kwargs[
"doReturnResults"] =
True
171 results = super(MeasureCrosstalkTask, cls).
parseAndRun(*args, **kwargs)
172 task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
173 resultList = [rr.result
for rr
in results.resultList]
174 if results.parsedCmd.dumpRatios:
176 pickle.dump(resultList, open(results.parsedCmd.dumpRatios,
"wb"))
177 coeff, coeffErr, coeffNum = task.reduce(resultList)
183 calib.coeffErr = coeffErr
184 calib.coeffNum = coeffNum
186 outputFileName = results.parsedCmd.outputFileName
187 if outputFileName
is not None:
188 butler = results.parsedCmd.butler
189 dataId = results.parsedCmd.id.idList[0]
192 det = butler.get(
'raw', dataId).getDetector()
193 calib._detectorName = det.getName()
194 calib._detectorSerial = det.getSerial()
195 calib.nAmp = len(det)
196 calib.hasCrosstalk =
True
197 calib.writeText(outputFileName +
".yaml")
199 provenance.calibType =
'CROSSTALK'
200 provenance._detectorName = det.getName()
201 provenance.fromDataIds(results.parsedCmd.id.idList)
202 provenance.writeText(outputFileName +
'_prov.yaml')
211 def _getConfigName(self):
212 """Disable config output."""
215 def _getMetadataName(self):
216 """Disable metdata output."""
220 """Get crosstalk ratios for detector.
224 dataRef : `lsst.daf.peristence.ButlerDataRef`
225 Data references for detectors to process.
229 ratios : `list` of `list` of `numpy.ndarray`
230 A matrix of pixel arrays.
233 if not self.config.doRerunIsr:
235 exposure = dataRef.get(
"postISRCCD")
240 exposure = self.isr.
runDataRef(dataRef).exposure
242 dataId = dataRef.dataId
243 return self.
run(exposure, dataId=dataId)
245 def run(self, exposure, dataId=None):
246 """Extract and return cross talk ratios for an exposure.
250 exposure : `lsst.afw.image.Exposure`
251 Image data to measure crosstalk ratios from.
253 Optional data ID for the exposure to process; used for logging.
257 ratios : `list` of `list` of `numpy.ndarray`
258 A matrix of pixel arrays.
261 self.log.info(
"Extracted %d pixels from %s",
262 sum(len(jj)
for ii
in ratios
for jj
in ii
if jj
is not None), dataId)
266 """Extract crosstalk ratios between different amplifiers.
268 For pixels above ``threshold``, we calculate the ratio between
269 each background-subtracted target amp and the source amp. We
270 return a list of ratios for each pixel for each target/source
271 combination, as a matrix of lists.
275 exposure : `lsst.afw.image.Exposure`
276 Exposure for which to measure crosstalk.
277 threshold : `float`, optional
278 Lower limit on pixels for which we measure crosstalk.
279 badPixels : `list` of `str`, optional
280 Mask planes indicating a pixel is bad.
284 ratios : `list` of `list` of `numpy.ndarray`
285 A matrix of pixel arrays. ``ratios[i][j]`` is an array of
286 the fraction of the ``j``-th amp present on the ``i``-th amp.
287 The value is `None` for the diagonal elements.
291 This has been moved into MeasureCrosstalkTask to allow for easier
294 The lsstDebug.Info() method can be rewritten for __name__ =
295 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
297 debug.display['extract'] : `bool`
298 Display the exposure under consideration, with the pixels used
299 for crosstalk measurement indicated by the DETECTED mask plane.
300 debug.display['pixels'] : `bool`
301 Display a plot of the ratio calculated for each pixel used in this
302 exposure, split by amplifier pairs. The median value is listed
305 if threshold
is None:
306 threshold = self.config.threshold
307 if badPixels
is None:
308 badPixels = list(self.config.badMask)
310 mi = exposure.getMaskedImage()
312 detected = mi.getMask().getPlaneBitMask(
"DETECTED")
313 bad = mi.getMask().getPlaneBitMask(badPixels)
314 bg = self.
calib.calculateBackground(mi, badPixels + [
"DETECTED"])
318 ccd = exposure.getDetector()
319 ratios = [[
None for iAmp
in ccd]
for jAmp
in ccd]
321 for ii, iAmp
in enumerate(ccd):
322 iImage = mi[iAmp.getBBox()]
323 iMask = iImage.mask.array
324 select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array)
325 for jj, jAmp
in enumerate(ccd):
328 jImage = self.
calib.extractAmp(mi.image, jAmp, iAmp, isTrimmed=self.config.isTrimmed)
329 ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select]
330 self.
debugPixels(
'pixels', iImage.image.array[select], jImage.array[select] - bg, ii, jj)
334 """Combine ratios to produce crosstalk coefficients.
338 ratioList : `list` of `list` of `list` of `numpy.ndarray`
339 A list of matrices of arrays; a list of results from
340 `extractCrosstalkRatios`.
344 coeff : `numpy.ndarray`
345 Crosstalk coefficients.
346 coeffErr : `numpy.ndarray`
347 Crosstalk coefficient errors.
348 coeffNum : `numpy.ndarray`
349 Number of pixels used for crosstalk measurement.
354 Raised if there is no crosstalk data available.
358 The lsstDebug.Info() method can be rewritten for __name__ =
359 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
361 debug.display['reduce'] : `bool`
362 Display a histogram of the combined ratio measurements for
363 a pair of source/target amplifiers from all input
374 assert len(rr) == numAmps
375 assert all(len(xx) == numAmps
for xx
in rr)
378 raise RuntimeError(
"Unable to measure crosstalk signal for any amplifier")
380 ratios = [[
None for jj
in range(numAmps)]
for ii
in range(numAmps)]
381 for ii, jj
in itertools.product(range(numAmps), range(numAmps)):
385 values = [rr[ii][jj]
for rr
in ratioList]
386 num = sum(len(vv)
for vv
in values)
388 self.log.warn(
"No values for matrix element %d,%d" % (ii, jj))
391 result = np.concatenate([vv
for vv
in values
if len(vv) > 0])
392 ratios[ii][jj] = result
395 self.config.rejSigma)
396 self.log.info(
"Coefficients:\n%s\n", coeff)
397 self.log.info(
"Errors:\n%s\n", coeffErr)
398 self.log.info(
"Numbers:\n%s\n", coeffNum)
399 return coeff, coeffErr, coeffNum
402 """Measure crosstalk coefficients from the ratios.
404 Given a list of ratios for each target/source amp combination,
405 we measure a sigma clipped mean and error.
407 The coefficient errors returned are the standard deviation of
408 the final set of clipped input ratios.
412 ratios : `list` of `list` of `numpy.ndarray`
413 Matrix of arrays of ratios.
415 Number of rejection iterations.
417 Rejection threshold (sigma).
421 coeff : `numpy.ndarray`
422 Crosstalk coefficients.
423 coeffErr : `numpy.ndarray`
424 Crosstalk coefficient errors.
425 coeffNum : `numpy.ndarray`
426 Number of pixels for each measurement.
430 This has been moved into MeasureCrosstalkTask to allow for easier
433 The lsstDebug.Info() method can be rewritten for __name__ =
434 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
436 debug.display['measure'] : `bool`
437 Display a histogram of the combined ratio measurements for
438 a pair of source/target amplifiers from the final set of
439 clipped input ratios.
442 rejIter = self.config.rejIter
444 rejSigma = self.config.rejSigma
446 numAmps = len(ratios)
447 assert all(len(rr) == numAmps
for rr
in ratios)
449 coeff = np.zeros((numAmps, numAmps))
450 coeffErr = np.zeros((numAmps, numAmps))
451 coeffNum = np.zeros((numAmps, numAmps), dtype=int)
453 for ii, jj
in itertools.product(range(numAmps), range(numAmps)):
457 values = np.array(ratios[ii][jj])
458 values = values[np.abs(values) < 1.0]
460 coeffNum[ii][jj] = len(values)
463 self.log.warn(
"No values for matrix element %d,%d" % (ii, jj))
464 coeff[ii][jj] = np.nan
465 coeffErr[ii][jj] = np.nan
468 for rej
in range(rejIter):
469 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
470 sigma = 0.741*(hi - lo)
471 good = np.abs(values - med) < rejSigma*sigma
472 if good.sum() == len(good):
474 values = values[good]
476 coeff[ii][jj] = np.mean(values)
477 coeffErr[ii][jj] = np.nan
if coeffNum[ii][jj] == 1
else np.std(values)
480 return coeff, coeffErr, coeffNum
483 """Utility function to examine the image being processed.
488 State of processing to view.
489 exposure : `lsst.afw.image.Exposure`
492 frame = getDebugFrame(self._display, stepname)
494 display = getDisplay(frame)
495 display.scale(
'asinh',
'zscale')
496 display.mtv(exposure)
498 prompt =
"Press Enter to continue: "
500 ans = input(prompt).lower()
501 if ans
in (
"",
"c",):
505 """Utility function to examine the CT ratio pixel values.
510 State of processing to view.
511 pixelsIn : `np.ndarray`
512 Pixel values from the potential crosstalk "source".
513 pixelsOut : `np.ndarray`
514 Pixel values from the potential crosstalk "victim".
516 Index of the source amplifier.
518 Index of the target amplifier.
520 frame = getDebugFrame(self._display, stepname)
522 if i == j
or len(pixelsIn) == 0
or len(pixelsOut) < 1:
524 import matplotlib.pyplot
as plot
525 figure = plot.figure(1)
528 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
529 axes.plot(pixelsIn, pixelsOut / pixelsIn,
'k+')
530 plot.xlabel(
"Source amplifier pixel value")
531 plot.ylabel(
"Measured pixel ratio")
532 plot.title(
"(Source %d -> Victim %d) median ratio: %f" %
533 (i, j, np.median(pixelsOut / pixelsIn)))
536 prompt =
"Press Enter to continue: "
538 ans = input(prompt).lower()
539 if ans
in (
"",
"c",):
544 """Utility function to examine the final CT ratio set.
549 State of processing to view.
550 ratios : `List` of `List` of `np.ndarray`
551 Array of measured CT ratios, indexed by source/victim
554 Index of the source amplifier.
556 Index of the target amplifier.
558 frame = getDebugFrame(self._display, stepname)
560 if i == j
or ratios
is None or len(ratios) < 1:
564 if RR
is None or len(RR) < 1:
569 import matplotlib.pyplot
as plot
570 figure = plot.figure(1)
572 plot.hist(x=RR, bins=
'auto', color=
'b', rwidth=0.9)
573 plot.xlabel(
"Measured pixel ratio")
574 plot.axvline(x=value, color=
"k")
575 plot.title(
"(Source %d -> Victim %d) clipped mean ratio: %f" % (i, j, value))
578 prompt =
"Press Enter to continue: "
580 ans = input(prompt).lower()
581 if ans
in (
"",
"c",):