lsst.ip.isr  13.0-15-g0af5a6c+28
measureCrosstalk.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2017 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 """
23 Measure intra-CCD crosstalk coefficients.
24 """
25 from __future__ import absolute_import, division, print_function
26 
27 __all__ = ["extractCrosstalkRatios", "measureCrosstalkCoefficients",
28  "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
29 
30 from builtins import range
31 
32 import itertools
33 import numpy as np
34 
35 from lsst.afw.detection import FootprintSet, Threshold
36 from lsst.pex.config import Config, Field, ListField, ConfigurableField
37 from lsst.pipe.base import CmdLineTask
38 
39 from .crosstalk import calculateBackground, extractAmp
40 from .isrTask import IsrTask
41 
42 
43 def extractCrosstalkRatios(exposure, threshold=30000, badPixels=["SAT", "BAD", "INTRP"]):
44  """Extract crosstalk ratios between different amplifiers
45 
46  For pixels above ``threshold``, we calculate the ratio between each
47  target amp and source amp. We return a list of ratios for each pixel
48  for each target/source combination, as a matrix of lists.
49 
50  Parameters
51  ----------
52  exposure : `lsst.afw.image.Exposure`
53  Exposure for which to measure crosstalk.
54  threshold : `float`
55  Lower limit on pixels for which we measure crosstalk.
56  badPixels : `list` of `str`
57  Mask planes indicating a pixel is bad.
58 
59  Returns
60  -------
61  ratios : `list` of `list` of `numpy.ndarray`
62  A matrix of pixel arrays. ``ratios[i][j]`` is an array of
63  the fraction of the ``j``-th amp present on the ``i``-th amp.
64  The value is `None` for the diagonal elements.
65  """
66  mi = exposure.getMaskedImage()
67  FootprintSet(mi, Threshold(threshold), "DETECTED")
68  detected = mi.getMask().getPlaneBitMask("DETECTED")
69  bad = mi.getMask().getPlaneBitMask(badPixels)
70  bg = calculateBackground(mi, badPixels + ["DETECTED"])
71 
72  ccd = exposure.getDetector()
73 
74  ratios = [[None for iAmp in ccd] for jAmp in ccd]
75 
76  for ii, iAmp in enumerate(ccd):
77  iImage = mi.Factory(mi, iAmp.getBBox())
78  iMask = iImage.getMask().getArray()
79  select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.getImage().getArray())
80  for jj, jAmp in enumerate(ccd):
81  if ii == jj:
82  continue
83  jImage = extractAmp(mi.getImage(), jAmp, iAmp.getReadoutCorner())
84  ratios[jj][ii] = (jImage.getArray()[select] - bg)/iImage.getImage().getArray()[select]
85 
86  return ratios
87 
88 
89 def measureCrosstalkCoefficients(ratios, rejIter=3, rejSigma=2.0):
90  """Measure crosstalk coefficients from the ratios
91 
92  Given a list of ratios for each target/source amp combination,
93  we measure a robust mean and error.
94 
95  The coefficient errors returned are the (robust) standard deviation of
96  the input ratios.
97 
98  Parameters
99  ----------
100  ratios : `list` of `list` of `numpy.ndarray`
101  Matrix of arrays of ratios.
102  rejIter : `int`
103  Number of rejection iterations.
104  rejSigma : `float`
105  Rejection threshold (sigma).
106 
107  Returns
108  -------
109  coeff : `numpy.ndarray`
110  Crosstalk coefficients.
111  coeffErr : `numpy.ndarray`
112  Crosstalk coefficient errors.
113  coeffNum : `numpy.ndarray`
114  Number of pixels for each measurement.
115  """
116  numAmps = len(ratios)
117  assert all(len(rr) == numAmps for rr in ratios)
118 
119  coeff = np.zeros((numAmps, numAmps))
120  coeffErr = np.zeros((numAmps, numAmps))
121  coeffNum = np.zeros((numAmps, numAmps), dtype=int)
122 
123  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
124  if ii == jj:
125  continue
126  values = np.array(ratios[ii][jj])
127  values = values[np.abs(values) < 1.0] # Discard unreasonable values
128  for rej in range(rejIter):
129  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
130  sigma = 0.741*(hi - lo)
131  good = np.abs(values - med) < rejSigma*sigma
132  if good.sum() == len(good):
133  break
134  values = values[good]
135 
136  coeff[ii][jj] = np.mean(values)
137  coeffErr[ii][jj] = np.std(values)
138  coeffNum[ii][jj] = len(values)
139 
140  return coeff, coeffErr, coeffNum
141 
142 
144  """Configuration for MeasureCrosstalkTask"""
145  isr = ConfigurableField(target=IsrTask, doc="Instrument signature removal")
146  threshold = Field(dtype=float, default=30000, doc="Minimum level for which to measure crosstalk")
147  badMask = ListField(dtype=str, default=["SAT", "BAD", "INTRP"], doc="Mask planes to ignore")
148  rejIter = Field(dtype=int, default=3, doc="Number of rejection iterations")
149  rejSigma = Field(dtype=float, default=2.0, doc="Rejection threshold (sigma)")
150 
151  def setDefaults(self):
152  Config.setDefaults(self)
153  self.isr.doWrite = False
154  self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal
155 
156 
157 class MeasureCrosstalkTask(CmdLineTask):
158  """Measure intra-CCD crosstalk
159 
160  This Task behaves in a scatter-gather fashion:
161  * Scatter: get ratios for each CCD.
162  * Gather: combine ratios to produce crosstalk coefficients.
163  """
164  ConfigClass = MeasureCrosstalkConfig
165  _DefaultName = "measureCrosstalk"
166 
167  def __init__(self, *args, **kwargs):
168  CmdLineTask.__init__(self, *args, **kwargs)
169  self.makeSubtask("isr")
170 
171  @classmethod
172  def _makeArgumentParser(cls):
173  parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
174  parser.add_argument("--dump-ratios", dest="dumpRatios",
175  help="Name of pickle file to which to write crosstalk ratios")
176  return parser
177 
178  @classmethod
179  def parseAndRun(cls, *args, **kwargs):
180  """Implement scatter/gather
181 
182  Returns
183  -------
184  coeff : `numpy.ndarray`
185  Crosstalk coefficients.
186  coeffErr : `numpy.ndarray`
187  Crosstalk coefficient errors.
188  coeffNum : `numpy.ndarray`
189  Number of pixels used for crosstalk measurement.
190  """
191  kwargs["doReturnResults"] = True
192  results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs)
193  task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
194  resultList = [rr.result for rr in results.resultList]
195  if results.parsedCmd.dumpRatios:
196  import pickle
197  pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "w"))
198  return task.reduce(resultList)
199 
200  def run(self, dataRef):
201  """Get crosstalk ratios for CCD
202 
203  Parameters
204  ----------
205  dataRef : `lsst.daf.peristence.ButlerDataRef`
206  Data reference for CCD.
207 
208  Returns
209  -------
210  ratios : `list` of `list` of `numpy.ndarray`
211  A matrix of pixel arrays.
212  """
213  exposure = self.isr.runDataRef(dataRef).exposure
214  ratios = extractCrosstalkRatios(exposure, self.config.threshold, list(self.config.badMask))
215  self.log.info("Extracted %d pixels from %s",
216  sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataRef.dataId)
217  return ratios
218 
219  def reduce(self, ratioList):
220  """Combine ratios to produce crosstalk coefficients
221 
222  Parameters
223  ----------
224  ratioList : `list` of `list` of `list` of `numpy.ndarray`
225  A list of matrices of arrays; a list of results from
226  `extractCrosstalkRatios`.
227 
228  Returns
229  -------
230  coeff : `numpy.ndarray`
231  Crosstalk coefficients.
232  coeffErr : `numpy.ndarray`
233  Crosstalk coefficient errors.
234  coeffNum : `numpy.ndarray`
235  Number of pixels used for crosstalk measurement.
236  """
237  numAmps = len(ratioList[0])
238  assert all(len(rr) == numAmps for rr in ratioList)
239  assert all(all(len(xx) == numAmps for xx in rr) for rr in ratioList)
240  ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)]
241  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
242  if ii == jj:
243  result = []
244  else:
245  values = [rr[ii][jj] for rr in ratioList]
246  num = sum(len(vv) for vv in values)
247  if num == 0:
248  raise RuntimeError("No values for matrix element %d,%d" % (ii, jj))
249  result = np.concatenate([vv for vv in values if len(vv) > 0])
250  ratios[ii][jj] = result
251  coeff, coeffErr, coeffNum = measureCrosstalkCoefficients(ratios, self.config.rejIter,
252  self.config.rejSigma)
253  self.log.info("Coefficients:\n%s\n", coeff)
254  self.log.info("Errors:\n%s\n", coeffErr)
255  self.log.info("Numbers:\n%s\n", coeffNum)
256  return coeff, coeffErr, coeffNum
257 
258  def _getConfigName(self):
259  """Disable config output"""
260  return None
261 
262  def _getMetadataName(self):
263  """Disable metdata output"""
264  return None
def extractAmp(image, amp, corner)
Definition: crosstalk.py:78
def measureCrosstalkCoefficients(ratios, rejIter=3, rejSigma=2.0)
def calculateBackground(mi, badPixels=["BAD"])
Definition: crosstalk.py:107
def extractCrosstalkRatios(exposure, threshold=30000, badPixels=["SAT", BAD, INTRP)