lsst.ip.isr  15.0-5-g23e394c+7
measureCrosstalk.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2017 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 """
23 Measure intra-CCD crosstalk coefficients.
24 """
25 
26 __all__ = ["extractCrosstalkRatios", "measureCrosstalkCoefficients",
27  "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
28 
29 
30 import itertools
31 import numpy as np
32 
33 from lsst.afw.detection import FootprintSet, Threshold
34 from lsst.pex.config import Config, Field, ListField, ConfigurableField
35 from lsst.pipe.base import CmdLineTask
36 
37 from .crosstalk import calculateBackground, extractAmp
38 from .isrTask import IsrTask
39 
40 
41 def extractCrosstalkRatios(exposure, threshold=30000, badPixels=["SAT", "BAD", "INTRP"]):
42  """Extract crosstalk ratios between different amplifiers
43 
44  For pixels above ``threshold``, we calculate the ratio between each
45  target amp and source amp. We return a list of ratios for each pixel
46  for each target/source combination, as a matrix of lists.
47 
48  Parameters
49  ----------
50  exposure : `lsst.afw.image.Exposure`
51  Exposure for which to measure crosstalk.
52  threshold : `float`
53  Lower limit on pixels for which we measure crosstalk.
54  badPixels : `list` of `str`
55  Mask planes indicating a pixel is bad.
56 
57  Returns
58  -------
59  ratios : `list` of `list` of `numpy.ndarray`
60  A matrix of pixel arrays. ``ratios[i][j]`` is an array of
61  the fraction of the ``j``-th amp present on the ``i``-th amp.
62  The value is `None` for the diagonal elements.
63  """
64  mi = exposure.getMaskedImage()
65  FootprintSet(mi, Threshold(threshold), "DETECTED")
66  detected = mi.getMask().getPlaneBitMask("DETECTED")
67  bad = mi.getMask().getPlaneBitMask(badPixels)
68  bg = calculateBackground(mi, badPixels + ["DETECTED"])
69 
70  ccd = exposure.getDetector()
71 
72  ratios = [[None for iAmp in ccd] for jAmp in ccd]
73 
74  for ii, iAmp in enumerate(ccd):
75  iImage = mi.Factory(mi, iAmp.getBBox())
76  iMask = iImage.getMask().getArray()
77  select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.getImage().getArray())
78  for jj, jAmp in enumerate(ccd):
79  if ii == jj:
80  continue
81  jImage = extractAmp(mi.getImage(), jAmp, iAmp.getReadoutCorner())
82  ratios[jj][ii] = (jImage.getArray()[select] - bg)/iImage.getImage().getArray()[select]
83 
84  return ratios
85 
86 
87 def measureCrosstalkCoefficients(ratios, rejIter=3, rejSigma=2.0):
88  """Measure crosstalk coefficients from the ratios
89 
90  Given a list of ratios for each target/source amp combination,
91  we measure a robust mean and error.
92 
93  The coefficient errors returned are the (robust) standard deviation of
94  the input ratios.
95 
96  Parameters
97  ----------
98  ratios : `list` of `list` of `numpy.ndarray`
99  Matrix of arrays of ratios.
100  rejIter : `int`
101  Number of rejection iterations.
102  rejSigma : `float`
103  Rejection threshold (sigma).
104 
105  Returns
106  -------
107  coeff : `numpy.ndarray`
108  Crosstalk coefficients.
109  coeffErr : `numpy.ndarray`
110  Crosstalk coefficient errors.
111  coeffNum : `numpy.ndarray`
112  Number of pixels for each measurement.
113  """
114  numAmps = len(ratios)
115  assert all(len(rr) == numAmps for rr in ratios)
116 
117  coeff = np.zeros((numAmps, numAmps))
118  coeffErr = np.zeros((numAmps, numAmps))
119  coeffNum = np.zeros((numAmps, numAmps), dtype=int)
120 
121  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
122  if ii == jj:
123  continue
124  values = np.array(ratios[ii][jj])
125  values = values[np.abs(values) < 1.0] # Discard unreasonable values
126  for rej in range(rejIter):
127  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
128  sigma = 0.741*(hi - lo)
129  good = np.abs(values - med) < rejSigma*sigma
130  if good.sum() == len(good):
131  break
132  values = values[good]
133 
134  coeff[ii][jj] = np.mean(values)
135  coeffErr[ii][jj] = np.std(values)
136  coeffNum[ii][jj] = len(values)
137 
138  return coeff, coeffErr, coeffNum
139 
140 
142  """Configuration for MeasureCrosstalkTask"""
143  isr = ConfigurableField(target=IsrTask, doc="Instrument signature removal")
144  threshold = Field(dtype=float, default=30000, doc="Minimum level for which to measure crosstalk")
145  badMask = ListField(dtype=str, default=["SAT", "BAD", "INTRP"], doc="Mask planes to ignore")
146  rejIter = Field(dtype=int, default=3, doc="Number of rejection iterations")
147  rejSigma = Field(dtype=float, default=2.0, doc="Rejection threshold (sigma)")
148 
149  def setDefaults(self):
150  Config.setDefaults(self)
151  self.isr.doWrite = False
152  self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal
153 
154 
155 class MeasureCrosstalkTask(CmdLineTask):
156  """Measure intra-CCD crosstalk
157 
158  This Task behaves in a scatter-gather fashion:
159  * Scatter: get ratios for each CCD.
160  * Gather: combine ratios to produce crosstalk coefficients.
161  """
162  ConfigClass = MeasureCrosstalkConfig
163  _DefaultName = "measureCrosstalk"
164 
165  def __init__(self, *args, **kwargs):
166  CmdLineTask.__init__(self, *args, **kwargs)
167  self.makeSubtask("isr")
168 
169  @classmethod
170  def _makeArgumentParser(cls):
171  parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
172  parser.add_argument("--dump-ratios", dest="dumpRatios",
173  help="Name of pickle file to which to write crosstalk ratios")
174  return parser
175 
176  @classmethod
177  def parseAndRun(cls, *args, **kwargs):
178  """Implement scatter/gather
179 
180  Returns
181  -------
182  coeff : `numpy.ndarray`
183  Crosstalk coefficients.
184  coeffErr : `numpy.ndarray`
185  Crosstalk coefficient errors.
186  coeffNum : `numpy.ndarray`
187  Number of pixels used for crosstalk measurement.
188  """
189  kwargs["doReturnResults"] = True
190  results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs)
191  task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
192  resultList = [rr.result for rr in results.resultList]
193  if results.parsedCmd.dumpRatios:
194  import pickle
195  pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "w"))
196  return task.reduce(resultList)
197 
198  def run(self, dataRef):
199  """Get crosstalk ratios for CCD
200 
201  Parameters
202  ----------
203  dataRef : `lsst.daf.peristence.ButlerDataRef`
204  Data reference for CCD.
205 
206  Returns
207  -------
208  ratios : `list` of `list` of `numpy.ndarray`
209  A matrix of pixel arrays.
210  """
211  exposure = self.isr.runDataRef(dataRef).exposure
212  ratios = extractCrosstalkRatios(exposure, self.config.threshold, list(self.config.badMask))
213  self.log.info("Extracted %d pixels from %s",
214  sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataRef.dataId)
215  return ratios
216 
217  def reduce(self, ratioList):
218  """Combine ratios to produce crosstalk coefficients
219 
220  Parameters
221  ----------
222  ratioList : `list` of `list` of `list` of `numpy.ndarray`
223  A list of matrices of arrays; a list of results from
224  `extractCrosstalkRatios`.
225 
226  Returns
227  -------
228  coeff : `numpy.ndarray`
229  Crosstalk coefficients.
230  coeffErr : `numpy.ndarray`
231  Crosstalk coefficient errors.
232  coeffNum : `numpy.ndarray`
233  Number of pixels used for crosstalk measurement.
234  """
235  numAmps = len(ratioList[0])
236  assert all(len(rr) == numAmps for rr in ratioList)
237  assert all(all(len(xx) == numAmps for xx in rr) for rr in ratioList)
238  ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)]
239  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
240  if ii == jj:
241  result = []
242  else:
243  values = [rr[ii][jj] for rr in ratioList]
244  num = sum(len(vv) for vv in values)
245  if num == 0:
246  raise RuntimeError("No values for matrix element %d,%d" % (ii, jj))
247  result = np.concatenate([vv for vv in values if len(vv) > 0])
248  ratios[ii][jj] = result
249  coeff, coeffErr, coeffNum = measureCrosstalkCoefficients(ratios, self.config.rejIter,
250  self.config.rejSigma)
251  self.log.info("Coefficients:\n%s\n", coeff)
252  self.log.info("Errors:\n%s\n", coeffErr)
253  self.log.info("Numbers:\n%s\n", coeffNum)
254  return coeff, coeffErr, coeffNum
255 
256  def _getConfigName(self):
257  """Disable config output"""
258  return None
259 
260  def _getMetadataName(self):
261  """Disable metdata output"""
262  return None
def extractAmp(image, amp, corner)
Definition: crosstalk.py:84
def measureCrosstalkCoefficients(ratios, rejIter=3, rejSigma=2.0)
def calculateBackground(mi, badPixels=["BAD"])
Definition: crosstalk.py:113
def extractCrosstalkRatios(exposure, threshold=30000, badPixels=["SAT", BAD, INTRP)