Coverage for python/lsst/ip/isr/measureCrosstalk.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2017 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22"""
23Measure intra-detector crosstalk coefficients.
24"""
26__all__ = ["MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
29import itertools
30import numpy as np
32from lsstDebug import getDebugFrame
33from lsst.afw.detection import FootprintSet, Threshold
34from lsst.afw.display import getDisplay
35from lsst.daf.persistence.butlerExceptions import NoResults
36from lsst.pex.config import Config, Field, ListField, ConfigurableField
37from lsst.pipe.base import CmdLineTask, Struct
39from .crosstalk import CrosstalkCalib
40from .calibType import IsrProvenance
41from .isrTask import IsrTask
44class MeasureCrosstalkConfig(Config):
45 """Configuration for MeasureCrosstalkTask."""
46 isr = ConfigurableField(
47 target=IsrTask,
48 doc="Instrument signature removal task to use to process data."
49 )
50 threshold = Field(
51 dtype=float,
52 default=30000,
53 doc="Minimum level of source pixels for which to measure crosstalk."
54 )
55 doRerunIsr = Field(
56 dtype=bool,
57 default=True,
58 doc="Rerun the ISR, even if postISRCCD files are available?"
59 )
60 badMask = ListField(
61 dtype=str,
62 default=["SAT", "BAD", "INTRP"],
63 doc="Mask planes to ignore when identifying source pixels."
64 )
65 rejIter = Field(
66 dtype=int,
67 default=3,
68 doc="Number of rejection iterations for final coefficient calculation."
69 )
70 rejSigma = Field(
71 dtype=float,
72 default=2.0,
73 doc="Rejection threshold (sigma) for final coefficient calculation."
74 )
75 isTrimmed = Field(
76 dtype=bool,
77 default=True,
78 doc="Have the amplifiers been trimmed before measuring CT?"
79 )
81 def setDefaults(self):
82 Config.setDefaults(self)
83 # Set ISR processing to run up until we would be applying the CT
84 # correction. Applying subsequent stages may corrupt the signal.
85 self.isr.doWrite = False
86 self.isr.doOverscan = True
87 self.isr.doAssembleCcd = True
88 self.isr.doBias = True
89 self.isr.doVariance = False # This isn't used in the calculation below.
90 self.isr.doLinearize = True # This is the last ISR step we need.
91 self.isr.doCrosstalk = False
92 self.isr.doBrighterFatter = False
93 self.isr.doDark = False
94 self.isr.doStrayLight = False
95 self.isr.doFlat = False
96 self.isr.doFringe = False
97 self.isr.doApplyGains = False
98 self.isr.doDefect = True # Masking helps remove spurious pixels.
99 self.isr.doSaturationInterpolation = False
100 self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal.
103class MeasureCrosstalkTask(CmdLineTask):
104 """Measure intra-detector crosstalk.
106 Notes
107 -----
108 The crosstalk this method measures assumes that when a bright
109 pixel is found in one detector amplifier, all other detector
110 amplifiers may see an increase in the same pixel location
111 (relative to the readout amplifier) as these other pixels are read
112 out at the same time.
114 After processing each input exposure through a limited set of ISR
115 stages, bright unmasked pixels above the threshold are identified.
116 The potential CT signal is found by taking the ratio of the
117 appropriate background-subtracted pixel value on the other
118 amplifiers to the input value on the source amplifier. If the
119 source amplifier has a large number of bright pixels as well, the
120 background level may be elevated, leading to poor ratio
121 measurements.
123 The set of ratios found between each pair of amplifiers across all
124 input exposures is then gathered to produce the final CT
125 coefficients. The sigma-clipped mean and sigma are returned from
126 these sets of ratios, with the coefficient to supply to the ISR
127 CrosstalkTask() being the multiplicative inverse of these values.
128 """
129 ConfigClass = MeasureCrosstalkConfig
130 _DefaultName = "measureCrosstalk"
132 def __init__(self, *args, **kwargs):
133 CmdLineTask.__init__(self, *args, **kwargs)
134 self.makeSubtask("isr")
135 self.calib = CrosstalkCalib()
137 @classmethod
138 def _makeArgumentParser(cls):
139 parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
140 parser.add_argument("--crosstalkName",
141 help="Name for this set of crosstalk coefficients", default="Unknown")
142 parser.add_argument("--outputFileName",
143 help="Name of yaml file to which to write crosstalk coefficients")
144 parser.add_argument("--dump-ratios", dest="dumpRatios",
145 help="Name of pickle file to which to write crosstalk ratios")
146 return parser
148 @classmethod
149 def parseAndRun(cls, *args, **kwargs):
150 """Collate crosstalk results from multiple exposures.
152 Process all input exposures through runDataRef, construct
153 final measurements from the final list of results from each
154 input, and persist the output calibration.
156 This method will be deprecated as part of DM-24760.
158 Returns
159 -------
160 coeff : `numpy.ndarray`
161 Crosstalk coefficients.
162 coeffErr : `numpy.ndarray`
163 Crosstalk coefficient errors.
164 coeffNum : `numpy.ndarray`
165 Number of pixels used for crosstalk measurement.
166 calib : `lsst.ip.isr.CrosstalkCalib`
167 Crosstalk object created from the measurements.
169 """
170 kwargs["doReturnResults"] = True
171 results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs)
172 task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
173 resultList = [rr.result for rr in results.resultList]
174 if results.parsedCmd.dumpRatios:
175 import pickle
176 pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "wb"))
177 coeff, coeffErr, coeffNum = task.reduce(resultList)
179 calib = CrosstalkCalib()
180 provenance = IsrProvenance()
182 calib.coeffs = coeff
183 calib.coeffErr = coeffErr
184 calib.coeffNum = coeffNum
186 outputFileName = results.parsedCmd.outputFileName
187 if outputFileName is not None:
188 butler = results.parsedCmd.butler
189 dataId = results.parsedCmd.id.idList[0]
191 # Rework to use lsst.ip.isr.CrosstalkCalib.
192 det = butler.get('raw', dataId).getDetector()
193 calib._detectorName = det.getName()
194 calib._detectorSerial = det.getSerial()
195 calib.nAmp = len(det)
196 calib.hasCrosstalk = True
197 calib.writeText(outputFileName + ".yaml")
199 provenance.calibType = 'CROSSTALK'
200 provenance._detectorName = det.getName()
201 provenance.fromDataIds(results.parsedCmd.id.idList)
202 provenance.writeText(outputFileName + '_prov.yaml')
204 return Struct(
205 coeff=coeff,
206 coeffErr=coeffErr,
207 coeffNum=coeffNum,
208 calib=calib,
209 )
211 def _getConfigName(self):
212 """Disable config output."""
213 return None
215 def _getMetadataName(self):
216 """Disable metdata output."""
217 return None
219 def runDataRef(self, dataRef):
220 """Get crosstalk ratios for detector.
222 Parameters
223 ----------
224 dataRef : `lsst.daf.peristence.ButlerDataRef`
225 Data references for detectors to process.
227 Returns
228 -------
229 ratios : `list` of `list` of `numpy.ndarray`
230 A matrix of pixel arrays.
231 """
232 exposure = None
233 if not self.config.doRerunIsr:
234 try:
235 exposure = dataRef.get("postISRCCD")
236 except NoResults:
237 pass
239 if exposure is None:
240 exposure = self.isr.runDataRef(dataRef).exposure
242 dataId = dataRef.dataId
243 return self.run(exposure, dataId=dataId)
245 def run(self, exposure, dataId=None):
246 """Extract and return cross talk ratios for an exposure.
248 Parameters
249 ----------
250 exposure : `lsst.afw.image.Exposure`
251 Image data to measure crosstalk ratios from.
252 dataId :
253 Optional data ID for the exposure to process; used for logging.
255 Returns
256 -------
257 ratios : `list` of `list` of `numpy.ndarray`
258 A matrix of pixel arrays.
259 """
260 ratios = self.extractCrosstalkRatios(exposure)
261 self.log.info("Extracted %d pixels from %s",
262 sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataId)
263 return ratios
265 def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None):
266 """Extract crosstalk ratios between different amplifiers.
268 For pixels above ``threshold``, we calculate the ratio between
269 each background-subtracted target amp and the source amp. We
270 return a list of ratios for each pixel for each target/source
271 combination, as a matrix of lists.
273 Parameters
274 ----------
275 exposure : `lsst.afw.image.Exposure`
276 Exposure for which to measure crosstalk.
277 threshold : `float`, optional
278 Lower limit on pixels for which we measure crosstalk.
279 badPixels : `list` of `str`, optional
280 Mask planes indicating a pixel is bad.
282 Returns
283 -------
284 ratios : `list` of `list` of `numpy.ndarray`
285 A matrix of pixel arrays. ``ratios[i][j]`` is an array of
286 the fraction of the ``j``-th amp present on the ``i``-th amp.
287 The value is `None` for the diagonal elements.
289 Notes
290 -----
291 This has been moved into MeasureCrosstalkTask to allow for easier
292 debugging.
294 The lsstDebug.Info() method can be rewritten for __name__ =
295 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
297 debug.display['extract'] : `bool`
298 Display the exposure under consideration, with the pixels used
299 for crosstalk measurement indicated by the DETECTED mask plane.
300 debug.display['pixels'] : `bool`
301 Display a plot of the ratio calculated for each pixel used in this
302 exposure, split by amplifier pairs. The median value is listed
303 for reference.
304 """
305 if threshold is None:
306 threshold = self.config.threshold
307 if badPixels is None:
308 badPixels = list(self.config.badMask)
310 mi = exposure.getMaskedImage()
311 FootprintSet(mi, Threshold(threshold), "DETECTED")
312 detected = mi.getMask().getPlaneBitMask("DETECTED")
313 bad = mi.getMask().getPlaneBitMask(badPixels)
314 bg = self.calib.calculateBackground(mi, badPixels + ["DETECTED"])
316 self.debugView('extract', exposure)
318 ccd = exposure.getDetector()
319 ratios = [[None for iAmp in ccd] for jAmp in ccd]
321 for ii, iAmp in enumerate(ccd):
322 iImage = mi[iAmp.getBBox()]
323 iMask = iImage.mask.array
324 select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array)
325 for jj, jAmp in enumerate(ccd):
326 if ii == jj:
327 continue
328 jImage = self.calib.extractAmp(mi.image, jAmp, iAmp, isTrimmed=self.config.isTrimmed)
329 ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select]
330 self.debugPixels('pixels', iImage.image.array[select], jImage.array[select] - bg, ii, jj)
331 return ratios
333 def reduce(self, ratioList):
334 """Combine ratios to produce crosstalk coefficients.
336 Parameters
337 ----------
338 ratioList : `list` of `list` of `list` of `numpy.ndarray`
339 A list of matrices of arrays; a list of results from
340 `extractCrosstalkRatios`.
342 Returns
343 -------
344 coeff : `numpy.ndarray`
345 Crosstalk coefficients.
346 coeffErr : `numpy.ndarray`
347 Crosstalk coefficient errors.
348 coeffNum : `numpy.ndarray`
349 Number of pixels used for crosstalk measurement.
351 Raises
352 ------
353 RuntimeError
354 Raised if there is no crosstalk data available.
356 Notes
357 -----
358 The lsstDebug.Info() method can be rewritten for __name__ =
359 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
361 debug.display['reduce'] : `bool`
362 Display a histogram of the combined ratio measurements for
363 a pair of source/target amplifiers from all input
364 exposures/detectors.
365 """
366 numAmps = None
367 for rr in ratioList:
368 if rr is None:
369 continue
371 if numAmps is None:
372 numAmps = len(rr)
374 assert len(rr) == numAmps
375 assert all(len(xx) == numAmps for xx in rr)
377 if numAmps is None:
378 raise RuntimeError("Unable to measure crosstalk signal for any amplifier")
380 ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)]
381 for ii, jj in itertools.product(range(numAmps), range(numAmps)):
382 if ii == jj:
383 result = []
384 else:
385 values = [rr[ii][jj] for rr in ratioList]
386 num = sum(len(vv) for vv in values)
387 if num == 0:
388 self.log.warn("No values for matrix element %d,%d" % (ii, jj))
389 result = np.nan
390 else:
391 result = np.concatenate([vv for vv in values if len(vv) > 0])
392 ratios[ii][jj] = result
393 self.debugRatios('reduce', ratios, ii, jj)
394 coeff, coeffErr, coeffNum = self.measureCrosstalkCoefficients(ratios, self.config.rejIter,
395 self.config.rejSigma)
396 self.log.info("Coefficients:\n%s\n", coeff)
397 self.log.info("Errors:\n%s\n", coeffErr)
398 self.log.info("Numbers:\n%s\n", coeffNum)
399 return coeff, coeffErr, coeffNum
401 def measureCrosstalkCoefficients(self, ratios, rejIter=3, rejSigma=2.0):
402 """Measure crosstalk coefficients from the ratios.
404 Given a list of ratios for each target/source amp combination,
405 we measure a sigma clipped mean and error.
407 The coefficient errors returned are the standard deviation of
408 the final set of clipped input ratios.
410 Parameters
411 ----------
412 ratios : `list` of `list` of `numpy.ndarray`
413 Matrix of arrays of ratios.
414 rejIter : `int`
415 Number of rejection iterations.
416 rejSigma : `float`
417 Rejection threshold (sigma).
419 Returns
420 -------
421 coeff : `numpy.ndarray`
422 Crosstalk coefficients.
423 coeffErr : `numpy.ndarray`
424 Crosstalk coefficient errors.
425 coeffNum : `numpy.ndarray`
426 Number of pixels for each measurement.
428 Notes
429 -----
430 This has been moved into MeasureCrosstalkTask to allow for easier
431 debugging.
433 The lsstDebug.Info() method can be rewritten for __name__ =
434 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
436 debug.display['measure'] : `bool`
437 Display a histogram of the combined ratio measurements for
438 a pair of source/target amplifiers from the final set of
439 clipped input ratios.
440 """
441 if rejIter is None:
442 rejIter = self.config.rejIter
443 if rejSigma is None:
444 rejSigma = self.config.rejSigma
446 numAmps = len(ratios)
447 assert all(len(rr) == numAmps for rr in ratios)
449 coeff = np.zeros((numAmps, numAmps))
450 coeffErr = np.zeros((numAmps, numAmps))
451 coeffNum = np.zeros((numAmps, numAmps), dtype=int)
453 for ii, jj in itertools.product(range(numAmps), range(numAmps)):
454 if ii == jj:
455 values = [0.0]
456 else:
457 values = np.array(ratios[ii][jj])
458 values = values[np.abs(values) < 1.0] # Discard unreasonable values
460 coeffNum[ii][jj] = len(values)
462 if len(values) == 0:
463 self.log.warn("No values for matrix element %d,%d" % (ii, jj))
464 coeff[ii][jj] = np.nan
465 coeffErr[ii][jj] = np.nan
466 else:
467 if ii != jj:
468 for rej in range(rejIter):
469 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
470 sigma = 0.741*(hi - lo)
471 good = np.abs(values - med) < rejSigma*sigma
472 if good.sum() == len(good):
473 break
474 values = values[good]
476 coeff[ii][jj] = np.mean(values)
477 coeffErr[ii][jj] = np.nan if coeffNum[ii][jj] == 1 else np.std(values)
478 self.debugRatios('measure', ratios, ii, jj)
480 return coeff, coeffErr, coeffNum
482 def debugView(self, stepname, exposure):
483 """Utility function to examine the image being processed.
485 Parameters
486 ----------
487 stepname : `str`
488 State of processing to view.
489 exposure : `lsst.afw.image.Exposure`
490 Exposure to view.
491 """
492 frame = getDebugFrame(self._display, stepname)
493 if frame:
494 display = getDisplay(frame)
495 display.scale('asinh', 'zscale')
496 display.mtv(exposure)
498 prompt = "Press Enter to continue: "
499 while True:
500 ans = input(prompt).lower()
501 if ans in ("", "c",):
502 break
504 def debugPixels(self, stepname, pixelsIn, pixelsOut, i, j):
505 """Utility function to examine the CT ratio pixel values.
507 Parameters
508 ----------
509 stepname : `str`
510 State of processing to view.
511 pixelsIn : `np.ndarray`
512 Pixel values from the potential crosstalk "source".
513 pixelsOut : `np.ndarray`
514 Pixel values from the potential crosstalk "victim".
515 i : `int`
516 Index of the source amplifier.
517 j : `int`
518 Index of the target amplifier.
519 """
520 frame = getDebugFrame(self._display, stepname)
521 if frame:
522 if i == j or len(pixelsIn) == 0 or len(pixelsOut) < 1:
523 pass
524 import matplotlib.pyplot as plot
525 figure = plot.figure(1)
526 figure.clear()
528 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
529 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+')
530 plot.xlabel("Source amplifier pixel value")
531 plot.ylabel("Measured pixel ratio")
532 plot.title("(Source %d -> Victim %d) median ratio: %f" %
533 (i, j, np.median(pixelsOut / pixelsIn)))
534 figure.show()
536 prompt = "Press Enter to continue: "
537 while True:
538 ans = input(prompt).lower()
539 if ans in ("", "c",):
540 break
541 plot.close()
543 def debugRatios(self, stepname, ratios, i, j):
544 """Utility function to examine the final CT ratio set.
546 Parameters
547 ----------
548 stepname : `str`
549 State of processing to view.
550 ratios : `List` of `List` of `np.ndarray`
551 Array of measured CT ratios, indexed by source/victim
552 amplifier.
553 i : `int`
554 Index of the source amplifier.
555 j : `int`
556 Index of the target amplifier.
557 """
558 frame = getDebugFrame(self._display, stepname)
559 if frame:
560 if i == j or ratios is None or len(ratios) < 1:
561 pass
563 RR = ratios[i][j]
564 if RR is None or len(RR) < 1:
565 pass
567 value = np.mean(RR)
569 import matplotlib.pyplot as plot
570 figure = plot.figure(1)
571 figure.clear()
572 plot.hist(x=RR, bins='auto', color='b', rwidth=0.9)
573 plot.xlabel("Measured pixel ratio")
574 plot.axvline(x=value, color="k")
575 plot.title("(Source %d -> Victim %d) clipped mean ratio: %f" % (i, j, value))
576 figure.show()
578 prompt = "Press Enter to continue: "
579 while True:
580 ans = input(prompt).lower()
581 if ans in ("", "c",):
582 break
583 plot.close()