lsst.ip.isr  20.0.0-3-g2fa8bb8+5
measureCrosstalk.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2017 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 """
23 Measure intra-detector crosstalk coefficients.
24 """
25 
26 __all__ = ["MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
27 
28 
29 import itertools
30 import numpy as np
31 
32 from lsstDebug import getDebugFrame
33 from lsst.afw.detection import FootprintSet, Threshold
34 from lsst.afw.display import getDisplay
35 from lsst.daf.persistence.butlerExceptions import NoResults
36 from lsst.pex.config import Config, Field, ListField, ConfigurableField
37 from lsst.pipe.base import CmdLineTask, Struct
38 
39 from .crosstalk import CrosstalkCalib
40 from .calibType import IsrProvenance
41 from .isrTask import IsrTask
42 
43 
44 class MeasureCrosstalkConfig(Config):
45  """Configuration for MeasureCrosstalkTask."""
46  isr = ConfigurableField(
47  target=IsrTask,
48  doc="Instrument signature removal task to use to process data."
49  )
50  threshold = Field(
51  dtype=float,
52  default=30000,
53  doc="Minimum level of source pixels for which to measure crosstalk."
54  )
55  doRerunIsr = Field(
56  dtype=bool,
57  default=True,
58  doc="Rerun the ISR, even if postISRCCD files are available?"
59  )
60  badMask = ListField(
61  dtype=str,
62  default=["SAT", "BAD", "INTRP"],
63  doc="Mask planes to ignore when identifying source pixels."
64  )
65  rejIter = Field(
66  dtype=int,
67  default=3,
68  doc="Number of rejection iterations for final coefficient calculation."
69  )
70  rejSigma = Field(
71  dtype=float,
72  default=2.0,
73  doc="Rejection threshold (sigma) for final coefficient calculation."
74  )
75  isTrimmed = Field(
76  dtype=bool,
77  default=True,
78  doc="Have the amplifiers been trimmed before measuring CT?"
79  )
80 
81  def setDefaults(self):
82  Config.setDefaults(self)
83  # Set ISR processing to run up until we would be applying the CT
84  # correction. Applying subsequent stages may corrupt the signal.
85  self.isr.doWrite = False
86  self.isr.doOverscan = True
87  self.isr.doAssembleCcd = True
88  self.isr.doBias = True
89  self.isr.doVariance = False # This isn't used in the calculation below.
90  self.isr.doLinearize = True # This is the last ISR step we need.
91  self.isr.doCrosstalk = False
92  self.isr.doBrighterFatter = False
93  self.isr.doDark = False
94  self.isr.doStrayLight = False
95  self.isr.doFlat = False
96  self.isr.doFringe = False
97  self.isr.doApplyGains = False
98  self.isr.doDefect = True # Masking helps remove spurious pixels.
99  self.isr.doSaturationInterpolation = False
100  self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal.
101 
102 
103 class MeasureCrosstalkTask(CmdLineTask):
104  """Measure intra-detector crosstalk.
105 
106  Notes
107  -----
108  The crosstalk this method measures assumes that when a bright
109  pixel is found in one detector amplifier, all other detector
110  amplifiers may see an increase in the same pixel location
111  (relative to the readout amplifier) as these other pixels are read
112  out at the same time.
113 
114  After processing each input exposure through a limited set of ISR
115  stages, bright unmasked pixels above the threshold are identified.
116  The potential CT signal is found by taking the ratio of the
117  appropriate background-subtracted pixel value on the other
118  amplifiers to the input value on the source amplifier. If the
119  source amplifier has a large number of bright pixels as well, the
120  background level may be elevated, leading to poor ratio
121  measurements.
122 
123  The set of ratios found between each pair of amplifiers across all
124  input exposures is then gathered to produce the final CT
125  coefficients. The sigma-clipped mean and sigma are returned from
126  these sets of ratios, with the coefficient to supply to the ISR
127  CrosstalkTask() being the multiplicative inverse of these values.
128  """
129  ConfigClass = MeasureCrosstalkConfig
130  _DefaultName = "measureCrosstalk"
131 
132  def __init__(self, *args, **kwargs):
133  CmdLineTask.__init__(self, *args, **kwargs)
134  self.makeSubtask("isr")
136 
137  @classmethod
138  def _makeArgumentParser(cls):
139  parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
140  parser.add_argument("--crosstalkName",
141  help="Name for this set of crosstalk coefficients", default="Unknown")
142  parser.add_argument("--outputFileName",
143  help="Name of yaml file to which to write crosstalk coefficients")
144  parser.add_argument("--dump-ratios", dest="dumpRatios",
145  help="Name of pickle file to which to write crosstalk ratios")
146  return parser
147 
148  @classmethod
149  def parseAndRun(cls, *args, **kwargs):
150  """Collate crosstalk results from multiple exposures.
151 
152  Process all input exposures through runDataRef, construct
153  final measurements from the final list of results from each
154  input, and persist the output calibration.
155 
156  This method will be deprecated as part of DM-24760.
157 
158  Returns
159  -------
160  coeff : `numpy.ndarray`
161  Crosstalk coefficients.
162  coeffErr : `numpy.ndarray`
163  Crosstalk coefficient errors.
164  coeffNum : `numpy.ndarray`
165  Number of pixels used for crosstalk measurement.
166  calib : `lsst.ip.isr.CrosstalkCalib`
167  Crosstalk object created from the measurements.
168 
169  """
170  kwargs["doReturnResults"] = True
171  results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs)
172  task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
173  resultList = [rr.result for rr in results.resultList]
174  if results.parsedCmd.dumpRatios:
175  import pickle
176  pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "wb"))
177  coeff, coeffErr, coeffNum = task.reduce(resultList)
178 
179  calib = CrosstalkCalib()
180  provenance = IsrProvenance()
181 
182  calib.coeffs = coeff
183  calib.coeffErr = coeffErr
184  calib.coeffNum = coeffNum
185 
186  outputFileName = results.parsedCmd.outputFileName
187  if outputFileName is not None:
188  butler = results.parsedCmd.butler
189  dataId = results.parsedCmd.id.idList[0]
190 
191  # Rework to use lsst.ip.isr.CrosstalkCalib.
192  det = butler.get('raw', dataId).getDetector()
193  calib._detectorName = det.getName()
194  calib._detectorSerial = det.getSerial()
195  calib.nAmp = len(det)
196  calib.hasCrosstalk = True
197  calib.writeText(outputFileName + ".yaml")
198 
199  provenance.calibType = 'CROSSTALK'
200  provenance._detectorName = det.getName()
201  provenance.fromDataIds(results.parsedCmd.id.idList)
202  provenance.writeText(outputFileName + '_prov.yaml')
203 
204  return Struct(
205  coeff=coeff,
206  coeffErr=coeffErr,
207  coeffNum=coeffNum,
208  calib=calib,
209  )
210 
211  def _getConfigName(self):
212  """Disable config output."""
213  return None
214 
215  def _getMetadataName(self):
216  """Disable metdata output."""
217  return None
218 
219  def runDataRef(self, dataRef):
220  """Get crosstalk ratios for detector.
221 
222  Parameters
223  ----------
224  dataRef : `lsst.daf.peristence.ButlerDataRef`
225  Data references for detectors to process.
226 
227  Returns
228  -------
229  ratios : `list` of `list` of `numpy.ndarray`
230  A matrix of pixel arrays.
231  """
232  exposure = None
233  if not self.config.doRerunIsr:
234  try:
235  exposure = dataRef.get("postISRCCD")
236  except NoResults:
237  pass
238 
239  if exposure is None:
240  exposure = self.isr.runDataRef(dataRef).exposure
241 
242  dataId = dataRef.dataId
243  return self.run(exposure, dataId=dataId)
244 
245  def run(self, exposure, dataId=None):
246  """Extract and return cross talk ratios for an exposure.
247 
248  Parameters
249  ----------
250  exposure : `lsst.afw.image.Exposure`
251  Image data to measure crosstalk ratios from.
252  dataId :
253  Optional data ID for the exposure to process; used for logging.
254 
255  Returns
256  -------
257  ratios : `list` of `list` of `numpy.ndarray`
258  A matrix of pixel arrays.
259  """
260  ratios = self.extractCrosstalkRatios(exposure)
261  self.log.info("Extracted %d pixels from %s",
262  sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataId)
263  return ratios
264 
265  def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None):
266  """Extract crosstalk ratios between different amplifiers.
267 
268  For pixels above ``threshold``, we calculate the ratio between
269  each background-subtracted target amp and the source amp. We
270  return a list of ratios for each pixel for each target/source
271  combination, as a matrix of lists.
272 
273  Parameters
274  ----------
275  exposure : `lsst.afw.image.Exposure`
276  Exposure for which to measure crosstalk.
277  threshold : `float`, optional
278  Lower limit on pixels for which we measure crosstalk.
279  badPixels : `list` of `str`, optional
280  Mask planes indicating a pixel is bad.
281 
282  Returns
283  -------
284  ratios : `list` of `list` of `numpy.ndarray`
285  A matrix of pixel arrays. ``ratios[i][j]`` is an array of
286  the fraction of the ``j``-th amp present on the ``i``-th amp.
287  The value is `None` for the diagonal elements.
288 
289  Notes
290  -----
291  This has been moved into MeasureCrosstalkTask to allow for easier
292  debugging.
293 
294  The lsstDebug.Info() method can be rewritten for __name__ =
295  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
296 
297  debug.display['extract'] : `bool`
298  Display the exposure under consideration, with the pixels used
299  for crosstalk measurement indicated by the DETECTED mask plane.
300  debug.display['pixels'] : `bool`
301  Display a plot of the ratio calculated for each pixel used in this
302  exposure, split by amplifier pairs. The median value is listed
303  for reference.
304  """
305  if threshold is None:
306  threshold = self.config.threshold
307  if badPixels is None:
308  badPixels = list(self.config.badMask)
309 
310  mi = exposure.getMaskedImage()
311  FootprintSet(mi, Threshold(threshold), "DETECTED")
312  detected = mi.getMask().getPlaneBitMask("DETECTED")
313  bad = mi.getMask().getPlaneBitMask(badPixels)
314  bg = self.calib.calculateBackground(mi, badPixels + ["DETECTED"])
315 
316  self.debugView('extract', exposure)
317 
318  ccd = exposure.getDetector()
319  ratios = [[None for iAmp in ccd] for jAmp in ccd]
320 
321  for ii, iAmp in enumerate(ccd):
322  iImage = mi[iAmp.getBBox()]
323  iMask = iImage.mask.array
324  select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array)
325  for jj, jAmp in enumerate(ccd):
326  if ii == jj:
327  continue
328  jImage = self.calib.extractAmp(mi.image, jAmp, iAmp, isTrimmed=self.config.isTrimmed)
329  ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select]
330  self.debugPixels('pixels', iImage.image.array[select], jImage.array[select] - bg, ii, jj)
331  return ratios
332 
333  def reduce(self, ratioList):
334  """Combine ratios to produce crosstalk coefficients.
335 
336  Parameters
337  ----------
338  ratioList : `list` of `list` of `list` of `numpy.ndarray`
339  A list of matrices of arrays; a list of results from
340  `extractCrosstalkRatios`.
341 
342  Returns
343  -------
344  coeff : `numpy.ndarray`
345  Crosstalk coefficients.
346  coeffErr : `numpy.ndarray`
347  Crosstalk coefficient errors.
348  coeffNum : `numpy.ndarray`
349  Number of pixels used for crosstalk measurement.
350 
351  Raises
352  ------
353  RuntimeError
354  Raised if there is no crosstalk data available.
355 
356  Notes
357  -----
358  The lsstDebug.Info() method can be rewritten for __name__ =
359  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
360 
361  debug.display['reduce'] : `bool`
362  Display a histogram of the combined ratio measurements for
363  a pair of source/target amplifiers from all input
364  exposures/detectors.
365  """
366  numAmps = None
367  for rr in ratioList:
368  if rr is None:
369  continue
370 
371  if numAmps is None:
372  numAmps = len(rr)
373 
374  assert len(rr) == numAmps
375  assert all(len(xx) == numAmps for xx in rr)
376 
377  if numAmps is None:
378  raise RuntimeError("Unable to measure crosstalk signal for any amplifier")
379 
380  ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)]
381  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
382  if ii == jj:
383  result = []
384  else:
385  values = [rr[ii][jj] for rr in ratioList]
386  num = sum(len(vv) for vv in values)
387  if num == 0:
388  self.log.warn("No values for matrix element %d,%d" % (ii, jj))
389  result = np.nan
390  else:
391  result = np.concatenate([vv for vv in values if len(vv) > 0])
392  ratios[ii][jj] = result
393  self.debugRatios('reduce', ratios, ii, jj)
394  coeff, coeffErr, coeffNum = self.measureCrosstalkCoefficients(ratios, self.config.rejIter,
395  self.config.rejSigma)
396  self.log.info("Coefficients:\n%s\n", coeff)
397  self.log.info("Errors:\n%s\n", coeffErr)
398  self.log.info("Numbers:\n%s\n", coeffNum)
399  return coeff, coeffErr, coeffNum
400 
401  def measureCrosstalkCoefficients(self, ratios, rejIter=3, rejSigma=2.0):
402  """Measure crosstalk coefficients from the ratios.
403 
404  Given a list of ratios for each target/source amp combination,
405  we measure a sigma clipped mean and error.
406 
407  The coefficient errors returned are the standard deviation of
408  the final set of clipped input ratios.
409 
410  Parameters
411  ----------
412  ratios : `list` of `list` of `numpy.ndarray`
413  Matrix of arrays of ratios.
414  rejIter : `int`
415  Number of rejection iterations.
416  rejSigma : `float`
417  Rejection threshold (sigma).
418 
419  Returns
420  -------
421  coeff : `numpy.ndarray`
422  Crosstalk coefficients.
423  coeffErr : `numpy.ndarray`
424  Crosstalk coefficient errors.
425  coeffNum : `numpy.ndarray`
426  Number of pixels for each measurement.
427 
428  Notes
429  -----
430  This has been moved into MeasureCrosstalkTask to allow for easier
431  debugging.
432 
433  The lsstDebug.Info() method can be rewritten for __name__ =
434  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
435 
436  debug.display['measure'] : `bool`
437  Display a histogram of the combined ratio measurements for
438  a pair of source/target amplifiers from the final set of
439  clipped input ratios.
440  """
441  if rejIter is None:
442  rejIter = self.config.rejIter
443  if rejSigma is None:
444  rejSigma = self.config.rejSigma
445 
446  numAmps = len(ratios)
447  assert all(len(rr) == numAmps for rr in ratios)
448 
449  coeff = np.zeros((numAmps, numAmps))
450  coeffErr = np.zeros((numAmps, numAmps))
451  coeffNum = np.zeros((numAmps, numAmps), dtype=int)
452 
453  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
454  if ii == jj:
455  values = [0.0]
456  else:
457  values = np.array(ratios[ii][jj])
458  values = values[np.abs(values) < 1.0] # Discard unreasonable values
459 
460  coeffNum[ii][jj] = len(values)
461 
462  if len(values) == 0:
463  self.log.warn("No values for matrix element %d,%d" % (ii, jj))
464  coeff[ii][jj] = np.nan
465  coeffErr[ii][jj] = np.nan
466  else:
467  if ii != jj:
468  for rej in range(rejIter):
469  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
470  sigma = 0.741*(hi - lo)
471  good = np.abs(values - med) < rejSigma*sigma
472  if good.sum() == len(good):
473  break
474  values = values[good]
475 
476  coeff[ii][jj] = np.mean(values)
477  coeffErr[ii][jj] = np.nan if coeffNum[ii][jj] == 1 else np.std(values)
478  self.debugRatios('measure', ratios, ii, jj)
479 
480  return coeff, coeffErr, coeffNum
481 
482  def debugView(self, stepname, exposure):
483  """Utility function to examine the image being processed.
484 
485  Parameters
486  ----------
487  stepname : `str`
488  State of processing to view.
489  exposure : `lsst.afw.image.Exposure`
490  Exposure to view.
491  """
492  frame = getDebugFrame(self._display, stepname)
493  if frame:
494  display = getDisplay(frame)
495  display.scale('asinh', 'zscale')
496  display.mtv(exposure)
497 
498  prompt = "Press Enter to continue: "
499  while True:
500  ans = input(prompt).lower()
501  if ans in ("", "c",):
502  break
503 
504  def debugPixels(self, stepname, pixelsIn, pixelsOut, i, j):
505  """Utility function to examine the CT ratio pixel values.
506 
507  Parameters
508  ----------
509  stepname : `str`
510  State of processing to view.
511  pixelsIn : `np.ndarray`
512  Pixel values from the potential crosstalk "source".
513  pixelsOut : `np.ndarray`
514  Pixel values from the potential crosstalk "victim".
515  i : `int`
516  Index of the source amplifier.
517  j : `int`
518  Index of the target amplifier.
519  """
520  frame = getDebugFrame(self._display, stepname)
521  if frame:
522  if i == j or len(pixelsIn) == 0 or len(pixelsOut) < 1:
523  pass
524  import matplotlib.pyplot as plot
525  figure = plot.figure(1)
526  figure.clear()
527 
528  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
529  axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+')
530  plot.xlabel("Source amplifier pixel value")
531  plot.ylabel("Measured pixel ratio")
532  plot.title("(Source %d -> Victim %d) median ratio: %f" %
533  (i, j, np.median(pixelsOut / pixelsIn)))
534  figure.show()
535 
536  prompt = "Press Enter to continue: "
537  while True:
538  ans = input(prompt).lower()
539  if ans in ("", "c",):
540  break
541  plot.close()
542 
543  def debugRatios(self, stepname, ratios, i, j):
544  """Utility function to examine the final CT ratio set.
545 
546  Parameters
547  ----------
548  stepname : `str`
549  State of processing to view.
550  ratios : `List` of `List` of `np.ndarray`
551  Array of measured CT ratios, indexed by source/victim
552  amplifier.
553  i : `int`
554  Index of the source amplifier.
555  j : `int`
556  Index of the target amplifier.
557  """
558  frame = getDebugFrame(self._display, stepname)
559  if frame:
560  if i == j or ratios is None or len(ratios) < 1:
561  pass
562 
563  RR = ratios[i][j]
564  if RR is None or len(RR) < 1:
565  pass
566 
567  value = np.mean(RR)
568 
569  import matplotlib.pyplot as plot
570  figure = plot.figure(1)
571  figure.clear()
572  plot.hist(x=RR, bins='auto', color='b', rwidth=0.9)
573  plot.xlabel("Measured pixel ratio")
574  plot.axvline(x=value, color="k")
575  plot.title("(Source %d -> Victim %d) clipped mean ratio: %f" % (i, j, value))
576  figure.show()
577 
578  prompt = "Press Enter to continue: "
579  while True:
580  ans = input(prompt).lower()
581  if ans in ("", "c",):
582  break
583  plot.close()
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.extractCrosstalkRatios
def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None)
Definition: measureCrosstalk.py:265
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkConfig.setDefaults
def setDefaults(self)
Definition: measureCrosstalk.py:81
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.debugView
def debugView(self, stepname, exposure)
Definition: measureCrosstalk.py:482
lsst::ip::isr.crosstalk.CrosstalkCalib
Definition: crosstalk.py:40
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.__init__
def __init__(self, *args, **kwargs)
Definition: measureCrosstalk.py:132
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.debugPixels
def debugPixels(self, stepname, pixelsIn, pixelsOut, i, j)
Definition: measureCrosstalk.py:504
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.runDataRef
def runDataRef(self, dataRef)
Definition: measureCrosstalk.py:219
lsst::afw::display
lsst::daf::persistence::butlerExceptions
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.parseAndRun
def parseAndRun(cls, *args, **kwargs)
Definition: measureCrosstalk.py:149
lsst::afw::detection::FootprintSet
lsst::ip::isr.calibType.IsrProvenance
Definition: calibType.py:431
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.calib
calib
Definition: measureCrosstalk.py:135
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.measureCrosstalkCoefficients
def measureCrosstalkCoefficients(self, ratios, rejIter=3, rejSigma=2.0)
Definition: measureCrosstalk.py:401
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask
Definition: measureCrosstalk.py:103
lsst::afw::detection::Threshold
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkConfig.isr
isr
Definition: measureCrosstalk.py:46
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.debugRatios
def debugRatios(self, stepname, ratios, i, j)
Definition: measureCrosstalk.py:543
lsst::afw::detection
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.reduce
def reduce(self, ratioList)
Definition: measureCrosstalk.py:333
lsst::pipe::base
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkTask.run
def run(self, exposure, dataId=None)
Definition: measureCrosstalk.py:245
lsst::ip::isr.measureCrosstalk.MeasureCrosstalkConfig
Definition: measureCrosstalk.py:44