Coverage for python/lsst/ip/isr/measureCrosstalk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Measure intra-detector crosstalk coefficients.
24"""
26__all__ = ["MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
29import itertools
30import numpy as np
32from lsstDebug import getDebugFrame
33from lsst.afw.detection import FootprintSet, Threshold
34from lsst.afw.display import getDisplay
35from lsst.daf.persistence.butlerExceptions import NoResults
36from lsst.pex.config import Config, Field, ListField, ConfigurableField
37from lsst.pipe.base import CmdLineTask, Struct
39from .crosstalk import calculateBackground, extractAmp, writeCrosstalkCoeffs
40from .isrTask import IsrTask
43class MeasureCrosstalkConfig(Config):
44 """Configuration for MeasureCrosstalkTask."""
45 isr = ConfigurableField(
46 target=IsrTask,
47 doc="Instrument signature removal task to use to process data."
48 )
49 threshold = Field(
50 dtype=float,
51 default=30000,
52 doc="Minimum level of source pixels for which to measure crosstalk."
53 )
54 doRerunIsr = Field(
55 dtype=bool,
56 default=True,
57 doc="Rerun the ISR, even if postISRCCD files are available?"
58 )
59 badMask = ListField(
60 dtype=str,
61 default=["SAT", "BAD", "INTRP"],
62 doc="Mask planes to ignore when identifying source pixels."
63 )
64 rejIter = Field(
65 dtype=int,
66 default=3,
67 doc="Number of rejection iterations for final coefficient calculation."
68 )
69 rejSigma = Field(
70 dtype=float,
71 default=2.0,
72 doc="Rejection threshold (sigma) for final coefficient calculation."
73 )
74 isTrimmed = Field(
75 dtype=bool,
76 default=True,
77 doc="Have the amplifiers been trimmed before measuring CT?"
78 )
80 def setDefaults(self):
81 Config.setDefaults(self)
82 # Set ISR processing to run up until we would be applying the CT
83 # correction. Applying subsequent stages may corrupt the signal.
84 self.isr.doWrite = False
85 self.isr.doOverscan = True
86 self.isr.doAssembleCcd = True
87 self.isr.doBias = True
88 self.isr.doVariance = False # This isn't used in the calculation below.
89 self.isr.doLinearize = True # This is the last ISR step we need.
90 self.isr.doCrosstalk = False
91 self.isr.doBrighterFatter = False
92 self.isr.doDark = False
93 self.isr.doStrayLight = False
94 self.isr.doFlat = False
95 self.isr.doFringe = False
96 self.isr.doApplyGains = False
97 self.isr.doDefect = True # Masking helps remove spurious pixels.
98 self.isr.doSaturationInterpolation = False
99 self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal.
102class MeasureCrosstalkTask(CmdLineTask):
103 """Measure intra-detector crosstalk.
105 Notes
106 -----
107 The crosstalk this method measures assumes that when a bright
108 pixel is found in one detector amplifier, all other detector
109 amplifiers may see an increase in the same pixel location
110 (relative to the readout amplifier) as these other pixels are read
111 out at the same time.
113 After processing each input exposure through a limited set of ISR
114 stages, bright unmasked pixels above the threshold are identified.
115 The potential CT signal is found by taking the ratio of the
116 appropriate background-subtracted pixel value on the other
117 amplifiers to the input value on the source amplifier. If the
118 source amplifier has a large number of bright pixels as well, the
119 background level may be elevated, leading to poor ratio
120 measurements.
122 The set of ratios found between each pair of amplifiers across all
123 input exposures is then gathered to produce the final CT
124 coefficients. The sigma-clipped mean and sigma are returned from
125 these sets of ratios, with the coefficient to supply to the ISR
126 CrosstalkTask() being the multiplicative inverse of these values.
127 """
128 ConfigClass = MeasureCrosstalkConfig
129 _DefaultName = "measureCrosstalk"
131 def __init__(self, *args, **kwargs):
132 CmdLineTask.__init__(self, *args, **kwargs)
133 self.makeSubtask("isr")
135 @classmethod
136 def _makeArgumentParser(cls):
137 parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
138 parser.add_argument("--crosstalkName",
139 help="Name for this set of crosstalk coefficients", default="Unknown")
140 parser.add_argument("--outputFileName",
141 help="Name of yaml file to which to write crosstalk coefficients")
142 parser.add_argument("--dump-ratios", dest="dumpRatios",
143 help="Name of pickle file to which to write crosstalk ratios")
144 return parser
146 @classmethod
147 def parseAndRun(cls, *args, **kwargs):
148 """Implement scatter/gather
150 Returns
151 -------
152 coeff : `numpy.ndarray`
153 Crosstalk coefficients.
154 coeffErr : `numpy.ndarray`
155 Crosstalk coefficient errors.
156 coeffNum : `numpy.ndarray`
157 Number of pixels used for crosstalk measurement.
158 """
159 kwargs["doReturnResults"] = True
160 results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs)
161 task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
162 resultList = [rr.result for rr in results.resultList]
163 if results.parsedCmd.dumpRatios:
164 import pickle
165 pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "wb"))
166 coeff, coeffErr, coeffNum = task.reduce(resultList)
168 outputFileName = results.parsedCmd.outputFileName
169 if outputFileName is not None:
170 butler = results.parsedCmd.butler
171 dataId = results.parsedCmd.id.idList[0]
172 dataId["detector"] = butler.queryMetadata("raw", ["detector"], dataId)[0]
174 det = butler.get('raw', dataId).getDetector()
175 writeCrosstalkCoeffs(outputFileName, coeff, det=det,
176 crosstalkName=results.parsedCmd.crosstalkName, indent=2)
178 return Struct(
179 coeff=coeff,
180 coeffErr=coeffErr,
181 coeffNum=coeffNum
182 )
184 def _getConfigName(self):
185 """Disable config output."""
186 return None
188 def _getMetadataName(self):
189 """Disable metdata output."""
190 return None
192 def runDataRef(self, dataRef):
193 """Get crosstalk ratios for detector.
195 Parameters
196 ----------
197 dataRef : `lsst.daf.peristence.ButlerDataRef`
198 Data references for detectors to process.
200 Returns
201 -------
202 ratios : `list` of `list` of `numpy.ndarray`
203 A matrix of pixel arrays.
204 """
205 exposure = None
206 if not self.config.doRerunIsr:
207 try:
208 exposure = dataRef.get("postISRCCD")
209 except NoResults:
210 pass
212 if exposure is None:
213 exposure = self.isr.runDataRef(dataRef).exposure
215 dataId = dataRef.dataId
216 return self.run(exposure, dataId=dataId)
218 def run(self, exposure, dataId=None):
219 """Extract and return cross talk ratios for an exposure.
221 Parameters
222 ----------
223 exposure : `lsst.afw.image.Exposure`
224 Image data to measure crosstalk ratios from.
225 dataId :
226 Optional data ID for the exposure to process; used for logging.
228 Returns
229 -------
230 ratios : `list` of `list` of `numpy.ndarray`
231 A matrix of pixel arrays.
232 """
233 ratios = self.extractCrosstalkRatios(exposure)
234 self.log.info("Extracted %d pixels from %s",
235 sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataId)
236 return ratios
238 def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None):
239 """Extract crosstalk ratios between different amplifiers.
241 For pixels above ``threshold``, we calculate the ratio between
242 each background-subtracted target amp and the source amp. We
243 return a list of ratios for each pixel for each target/source
244 combination, as a matrix of lists.
246 Parameters
247 ----------
248 exposure : `lsst.afw.image.Exposure`
249 Exposure for which to measure crosstalk.
250 threshold : `float`, optional
251 Lower limit on pixels for which we measure crosstalk.
252 badPixels : `list` of `str`, optional
253 Mask planes indicating a pixel is bad.
255 Returns
256 -------
257 ratios : `list` of `list` of `numpy.ndarray`
258 A matrix of pixel arrays. ``ratios[i][j]`` is an array of
259 the fraction of the ``j``-th amp present on the ``i``-th amp.
260 The value is `None` for the diagonal elements.
262 Notes
263 -----
264 This has been moved into MeasureCrosstalkTask to allow for easier
265 debugging.
267 The lsstDebug.Info() method can be rewritten for __name__ =
268 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
270 debug.display['extract'] : `bool`
271 Display the exposure under consideration, with the pixels used
272 for crosstalk measurement indicated by the DETECTED mask plane.
273 debug.display['pixels'] : `bool`
274 Display a plot of the ratio calculated for each pixel used in this
275 exposure, split by amplifier pairs. The median value is listed
276 for reference.
277 """
278 if threshold is None:
279 threshold = self.config.threshold
280 if badPixels is None:
281 badPixels = list(self.config.badMask)
283 mi = exposure.getMaskedImage()
284 FootprintSet(mi, Threshold(threshold), "DETECTED")
285 detected = mi.getMask().getPlaneBitMask("DETECTED")
286 bad = mi.getMask().getPlaneBitMask(badPixels)
287 bg = calculateBackground(mi, badPixels + ["DETECTED"])
289 self.debugView('extract', exposure)
291 ccd = exposure.getDetector()
292 ratios = [[None for iAmp in ccd] for jAmp in ccd]
294 for ii, iAmp in enumerate(ccd):
295 iImage = mi[iAmp.getBBox()]
296 iMask = iImage.mask.array
297 select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array)
298 for jj, jAmp in enumerate(ccd):
299 if ii == jj:
300 continue
301 jImage = extractAmp(mi.image, jAmp, iAmp.getReadoutCorner(), isTrimmed=self.config.isTrimmed)
302 ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select]
303 self.debugPixels('pixels', iImage.image.array[select], jImage.array[select] - bg, ii, jj)
304 return ratios
306 def reduce(self, ratioList):
307 """Combine ratios to produce crosstalk coefficients.
309 Parameters
310 ----------
311 ratioList : `list` of `list` of `list` of `numpy.ndarray`
312 A list of matrices of arrays; a list of results from
313 `extractCrosstalkRatios`.
315 Returns
316 -------
317 coeff : `numpy.ndarray`
318 Crosstalk coefficients.
319 coeffErr : `numpy.ndarray`
320 Crosstalk coefficient errors.
321 coeffNum : `numpy.ndarray`
322 Number of pixels used for crosstalk measurement.
324 Raises
325 ------
326 RuntimeError
327 Raised if there is no crosstalk data available.
329 Notes
330 -----
331 The lsstDebug.Info() method can be rewritten for __name__ =
332 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
334 debug.display['reduce'] : `bool`
335 Display a histogram of the combined ratio measurements for
336 a pair of source/target amplifiers from all input
337 exposures/detectors.
338 """
339 numAmps = None
340 for rr in ratioList:
341 if rr is None:
342 continue
344 if numAmps is None:
345 numAmps = len(rr)
347 assert len(rr) == numAmps
348 assert all(len(xx) == numAmps for xx in rr)
350 if numAmps is None:
351 raise RuntimeError("Unable to measure crosstalk signal for any amplifier")
353 ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)]
354 for ii, jj in itertools.product(range(numAmps), range(numAmps)):
355 if ii == jj:
356 result = []
357 else:
358 values = [rr[ii][jj] for rr in ratioList]
359 num = sum(len(vv) for vv in values)
360 if num == 0:
361 self.log.warn("No values for matrix element %d,%d" % (ii, jj))
362 result = np.nan
363 else:
364 result = np.concatenate([vv for vv in values if len(vv) > 0])
365 ratios[ii][jj] = result
366 self.debugRatios('reduce', ratios, ii, jj)
367 coeff, coeffErr, coeffNum = self.measureCrosstalkCoefficients(ratios, self.config.rejIter,
368 self.config.rejSigma)
369 self.log.info("Coefficients:\n%s\n", coeff)
370 self.log.info("Errors:\n%s\n", coeffErr)
371 self.log.info("Numbers:\n%s\n", coeffNum)
372 return coeff, coeffErr, coeffNum
374 def measureCrosstalkCoefficients(self, ratios, rejIter=3, rejSigma=2.0):
375 """Measure crosstalk coefficients from the ratios.
377 Given a list of ratios for each target/source amp combination,
378 we measure a sigma clipped mean and error.
380 The coefficient errors returned are the standard deviation of
381 the final set of clipped input ratios.
383 Parameters
384 ----------
385 ratios : `list` of `list` of `numpy.ndarray`
386 Matrix of arrays of ratios.
387 rejIter : `int`
388 Number of rejection iterations.
389 rejSigma : `float`
390 Rejection threshold (sigma).
392 Returns
393 -------
394 coeff : `numpy.ndarray`
395 Crosstalk coefficients.
396 coeffErr : `numpy.ndarray`
397 Crosstalk coefficient errors.
398 coeffNum : `numpy.ndarray`
399 Number of pixels for each measurement.
401 Notes
402 -----
403 This has been moved into MeasureCrosstalkTask to allow for easier
404 debugging.
406 The lsstDebug.Info() method can be rewritten for __name__ =
407 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
409 debug.display['measure'] : `bool`
410 Display a histogram of the combined ratio measurements for
411 a pair of source/target amplifiers from the final set of
412 clipped input ratios.
413 """
414 if rejIter is None:
415 rejIter = self.config.rejIter
416 if rejSigma is None:
417 rejSigma = self.config.rejSigma
419 numAmps = len(ratios)
420 assert all(len(rr) == numAmps for rr in ratios)
422 coeff = np.zeros((numAmps, numAmps))
423 coeffErr = np.zeros((numAmps, numAmps))
424 coeffNum = np.zeros((numAmps, numAmps), dtype=int)
426 for ii, jj in itertools.product(range(numAmps), range(numAmps)):
427 if ii == jj:
428 values = [0.0]
429 else:
430 values = np.array(ratios[ii][jj])
431 values = values[np.abs(values) < 1.0] # Discard unreasonable values
433 coeffNum[ii][jj] = len(values)
435 if len(values) == 0:
436 self.log.warn("No values for matrix element %d,%d" % (ii, jj))
437 coeff[ii][jj] = np.nan
438 coeffErr[ii][jj] = np.nan
439 else:
440 if ii != jj:
441 for rej in range(rejIter):
442 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
443 sigma = 0.741*(hi - lo)
444 good = np.abs(values - med) < rejSigma*sigma
445 if good.sum() == len(good):
446 break
447 values = values[good]
449 coeff[ii][jj] = np.mean(values)
450 coeffErr[ii][jj] = np.nan if coeffNum[ii][jj] == 1 else np.std(values)
451 self.debugRatios('measure', ratios, ii, jj)
453 return coeff, coeffErr, coeffNum
455 def debugView(self, stepname, exposure):
456 """Utility function to examine the image being processed.
458 Parameters
459 ----------
460 stepname : `str`
461 State of processing to view.
462 exposure : `lsst.afw.image.Exposure`
463 Exposure to view.
464 """
465 frame = getDebugFrame(self._display, stepname)
466 if frame:
467 display = getDisplay(frame)
468 display.scale('asinh', 'zscale')
469 display.mtv(exposure)
471 prompt = "Press Enter to continue: "
472 while True:
473 ans = input(prompt).lower()
474 if ans in ("", "c",):
475 break
477 def debugPixels(self, stepname, pixelsIn, pixelsOut, i, j):
478 """Utility function to examine the CT ratio pixel values.
480 Parameters
481 ----------
482 stepname : `str`
483 State of processing to view.
484 pixelsIn : `np.ndarray`
485 Pixel values from the potential crosstalk "source".
486 pixelsOut : `np.ndarray`
487 Pixel values from the potential crosstalk "victim".
488 i : `int`
489 Index of the source amplifier.
490 j : `int`
491 Index of the target amplifier.
492 """
493 frame = getDebugFrame(self._display, stepname)
494 if frame:
495 if i == j or len(pixelsIn) == 0 or len(pixelsOut) < 1:
496 pass
497 import matplotlib.pyplot as plot
498 figure = plot.figure(1)
499 figure.clear()
501 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
502 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+')
503 plot.xlabel("Source amplifier pixel value")
504 plot.ylabel("Measured pixel ratio")
505 plot.title("(Source %d -> Victim %d) median ratio: %f" %
506 (i, j, np.median(pixelsOut / pixelsIn)))
507 figure.show()
509 prompt = "Press Enter to continue: "
510 while True:
511 ans = input(prompt).lower()
512 if ans in ("", "c",):
513 break
514 plot.close()
516 def debugRatios(self, stepname, ratios, i, j):
517 """Utility function to examine the final CT ratio set.
519 Parameters
520 ----------
521 stepname : `str`
522 State of processing to view.
523 ratios : `List` of `List` of `np.ndarray`
524 Array of measured CT ratios, indexed by source/victim
525 amplifier.
526 i : `int`
527 Index of the source amplifier.
528 j : `int`
529 Index of the target amplifier.
530 """
531 frame = getDebugFrame(self._display, stepname)
532 if frame:
533 if i == j or ratios is None or len(ratios) < 1:
534 pass
536 RR = ratios[i][j]
537 if RR is None or len(RR) < 1:
538 pass
540 value = np.mean(RR)
542 import matplotlib.pyplot as plot
543 figure = plot.figure(1)
544 figure.clear()
545 plot.hist(x=RR, bins='auto', color='b', rwidth=0.9)
546 plot.xlabel("Measured pixel ratio")
547 plot.axvline(x=value, color="k")
548 plot.title("(Source %d -> Victim %d) clipped mean ratio: %f" % (i, j, value))
549 figure.show()
551 prompt = "Press Enter to continue: "
552 while True:
553 ans = input(prompt).lower()
554 if ans in ("", "c",):
555 break
556 plot.close()