lsst.ip.isr  16.0-7-g2c7e7ec+5
measureCrosstalk.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2017 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 """
23 Measure intra-CCD crosstalk coefficients.
24 """
25 
26 __all__ = ["extractCrosstalkRatios", "measureCrosstalkCoefficients",
27  "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
28 
29 
30 import itertools
31 import numpy as np
32 
33 from lsst.afw.detection import FootprintSet, Threshold
34 from lsst.daf.persistence.butlerExceptions import NoResults
35 from lsst.pex.config import Config, Field, ListField, ConfigurableField
36 from lsst.pipe.base import CmdLineTask, Struct
37 
38 from .crosstalk import calculateBackground, extractAmp, writeCrosstalkCoeffs
39 from .isrTask import IsrTask
40 
41 
42 def extractCrosstalkRatios(exposure, threshold=30000, badPixels=["SAT", "BAD", "INTRP"]):
43  """Extract crosstalk ratios between different amplifiers
44 
45  For pixels above ``threshold``, we calculate the ratio between each
46  target amp and source amp. We return a list of ratios for each pixel
47  for each target/source combination, as a matrix of lists.
48 
49  Parameters
50  ----------
51  exposure : `lsst.afw.image.Exposure`
52  Exposure for which to measure crosstalk.
53  threshold : `float`
54  Lower limit on pixels for which we measure crosstalk.
55  badPixels : `list` of `str`
56  Mask planes indicating a pixel is bad.
57 
58  Returns
59  -------
60  ratios : `list` of `list` of `numpy.ndarray`
61  A matrix of pixel arrays. ``ratios[i][j]`` is an array of
62  the fraction of the ``j``-th amp present on the ``i``-th amp.
63  The value is `None` for the diagonal elements.
64  """
65  mi = exposure.getMaskedImage()
66  FootprintSet(mi, Threshold(threshold), "DETECTED")
67  detected = mi.getMask().getPlaneBitMask("DETECTED")
68  bad = mi.getMask().getPlaneBitMask(badPixels)
69  bg = calculateBackground(mi, badPixels + ["DETECTED"])
70 
71  ccd = exposure.getDetector()
72 
73  ratios = [[None for iAmp in ccd] for jAmp in ccd]
74 
75  for ii, iAmp in enumerate(ccd):
76  iImage = mi[iAmp.getBBox()]
77  iMask = iImage.mask.array
78  select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array)
79  for jj, jAmp in enumerate(ccd):
80  if ii == jj:
81  continue
82  jImage = extractAmp(mi.image, jAmp, iAmp.getReadoutCorner(), isTrimmed=True)
83  ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select]
84 
85  return ratios
86 
87 
88 def measureCrosstalkCoefficients(ratios, rejIter=3, rejSigma=2.0):
89  """Measure crosstalk coefficients from the ratios
90 
91  Given a list of ratios for each target/source amp combination,
92  we measure a robust mean and error.
93 
94  The coefficient errors returned are the (robust) standard deviation of
95  the input ratios.
96 
97  Parameters
98  ----------
99  ratios : `list` of `list` of `numpy.ndarray`
100  Matrix of arrays of ratios.
101  rejIter : `int`
102  Number of rejection iterations.
103  rejSigma : `float`
104  Rejection threshold (sigma).
105 
106  Returns
107  -------
108  coeff : `numpy.ndarray`
109  Crosstalk coefficients.
110  coeffErr : `numpy.ndarray`
111  Crosstalk coefficient errors.
112  coeffNum : `numpy.ndarray`
113  Number of pixels for each measurement.
114  """
115  numAmps = len(ratios)
116  assert all(len(rr) == numAmps for rr in ratios)
117 
118  coeff = np.zeros((numAmps, numAmps))
119  coeffErr = np.zeros((numAmps, numAmps))
120  coeffNum = np.zeros((numAmps, numAmps), dtype=int)
121 
122  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
123  if ii == jj:
124  values = [0.0]
125  else:
126  values = np.array(ratios[ii][jj])
127  values = values[np.abs(values) < 1.0] # Discard unreasonable values
128 
129  coeffNum[ii][jj] = len(values)
130 
131  if len(values) == 0:
132  coeff[ii][jj] = np.nan
133  coeffErr[ii][jj] = np.nan
134  else:
135  if ii != jj:
136  for rej in range(rejIter):
137  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
138  sigma = 0.741*(hi - lo)
139  good = np.abs(values - med) < rejSigma*sigma
140  if good.sum() == len(good):
141  break
142  values = values[good]
143 
144  coeff[ii][jj] = np.mean(values)
145  coeffErr[ii][jj] = np.nan if coeffNum[ii][jj] == 1 else np.std(values)
146 
147  return coeff, coeffErr, coeffNum
148 
149 
151  """Configuration for MeasureCrosstalkTask"""
152  isr = ConfigurableField(target=IsrTask, doc="Instrument signature removal")
153  threshold = Field(dtype=float, default=30000, doc="Minimum level for which to measure crosstalk")
154  doRerunIsr = Field(dtype=bool, default=True, doc="Rerun the ISR, even if postISRCCD files are available")
155  badMask = ListField(dtype=str, default=["SAT", "BAD", "INTRP"], doc="Mask planes to ignore")
156  rejIter = Field(dtype=int, default=3, doc="Number of rejection iterations")
157  rejSigma = Field(dtype=float, default=2.0, doc="Rejection threshold (sigma)")
158 
159  def setDefaults(self):
160  Config.setDefaults(self)
161  self.isr.doWrite = False
162  self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal
163 
164 
165 class MeasureCrosstalkTask(CmdLineTask):
166  """Measure intra-CCD crosstalk
167 
168  This Task behaves in a scatter-gather fashion:
169  * Scatter: get ratios for each CCD.
170  * Gather: combine ratios to produce crosstalk coefficients.
171  """
172  ConfigClass = MeasureCrosstalkConfig
173  _DefaultName = "measureCrosstalk"
174 
175  def __init__(self, *args, **kwargs):
176  CmdLineTask.__init__(self, *args, **kwargs)
177  self.makeSubtask("isr")
178 
179  @classmethod
180  def _makeArgumentParser(cls):
181  parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser()
182  parser.add_argument("--crosstalkName",
183  help="Name for this set of crosstalk coefficients", default="Unknown")
184  parser.add_argument("--outputFileName",
185  help="Name of yaml file to which to write crosstalk coefficients")
186  parser.add_argument("--dump-ratios", dest="dumpRatios",
187  help="Name of pickle file to which to write crosstalk ratios")
188  return parser
189 
190  @classmethod
191  def parseAndRun(cls, *args, **kwargs):
192  """Implement scatter/gather
193 
194  Returns
195  -------
196  coeff : `numpy.ndarray`
197  Crosstalk coefficients.
198  coeffErr : `numpy.ndarray`
199  Crosstalk coefficient errors.
200  coeffNum : `numpy.ndarray`
201  Number of pixels used for crosstalk measurement.
202  """
203  kwargs["doReturnResults"] = True
204  results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs)
205  task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log)
206  resultList = [rr.result for rr in results.resultList]
207  if results.parsedCmd.dumpRatios:
208  import pickle
209  pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "wb"))
210  coeff, coeffErr, coeffNum = task.reduce(resultList)
211 
212  outputFileName = results.parsedCmd.outputFileName
213  if outputFileName is not None:
214  butler = results.parsedCmd.butler
215  dataId = results.parsedCmd.id.idList[0]
216  dataId["detector"] = butler.queryMetadata("raw", ["detector"], dataId)[0]
217 
218  det = butler.get('raw', dataId).getDetector()
219  writeCrosstalkCoeffs(outputFileName, coeff, det=det,
220  crosstalkName=results.parsedCmd.crosstalkName, indent=2)
221 
222  return Struct(
223  coeff=coeff,
224  coeffErr=coeffErr,
225  coeffNum=coeffNum
226  )
227 
228  def runDataRef(self, dataRef):
229  """Get crosstalk ratios for CCD
230 
231  Parameters
232  ----------
233  dataRef : `lsst.daf.peristence.ButlerDataRef`
234  Data reference for CCD.
235 
236  Returns
237  -------
238  ratios : `list` of `list` of `numpy.ndarray`
239  A matrix of pixel arrays.
240  """
241  exposure = None
242  if not self.config.doRerunIsr:
243  try:
244  exposure = dataRef.get("postISRCCD")
245  except NoResults:
246  pass
247 
248  if exposure is None:
249  exposure = self.isr.runDataRef(dataRef).exposure
250 
251  ratios = extractCrosstalkRatios(exposure, self.config.threshold, list(self.config.badMask))
252  self.log.info("Extracted %d pixels from %s",
253  sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataRef.dataId)
254  return ratios
255 
256  def reduce(self, ratioList):
257  """Combine ratios to produce crosstalk coefficients
258 
259  Parameters
260  ----------
261  ratioList : `list` of `list` of `list` of `numpy.ndarray`
262  A list of matrices of arrays; a list of results from
263  `extractCrosstalkRatios`.
264 
265  Returns
266  -------
267  coeff : `numpy.ndarray`
268  Crosstalk coefficients.
269  coeffErr : `numpy.ndarray`
270  Crosstalk coefficient errors.
271  coeffNum : `numpy.ndarray`
272  Number of pixels used for crosstalk measurement.
273  """
274  numAmps = None
275  for rr in ratioList:
276  if rr is None:
277  continue
278 
279  if numAmps is None:
280  numAmps = len(rr)
281 
282  assert len(rr) == numAmps
283  assert all(len(xx) == numAmps for xx in rr)
284 
285  if numAmps is None:
286  raise RuntimeError("Unable to measure crosstalk signal for any amplifier")
287 
288  ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)]
289  for ii, jj in itertools.product(range(numAmps), range(numAmps)):
290  if ii == jj:
291  result = []
292  else:
293  values = [rr[ii][jj] for rr in ratioList]
294  num = sum(len(vv) for vv in values)
295  if num == 0:
296  self.log.warn("No values for matrix element %d,%d" % (ii, jj))
297  result = np.nan
298  else:
299  result = np.concatenate([vv for vv in values if len(vv) > 0])
300  ratios[ii][jj] = result
301  coeff, coeffErr, coeffNum = measureCrosstalkCoefficients(ratios, self.config.rejIter,
302  self.config.rejSigma)
303  self.log.info("Coefficients:\n%s\n", coeff)
304  self.log.info("Errors:\n%s\n", coeffErr)
305  self.log.info("Numbers:\n%s\n", coeffNum)
306  return coeff, coeffErr, coeffNum
307 
308  def _getConfigName(self):
309  """Disable config output"""
310  return None
311 
312  def _getMetadataName(self):
313  """Disable metdata output"""
314  return None
def measureCrosstalkCoefficients(ratios, rejIter=3, rejSigma=2.0)
def calculateBackground(mi, badPixels=["BAD"])
Definition: crosstalk.py:116
def extractCrosstalkRatios(exposure, threshold=30000, badPixels=["SAT", BAD, INTRP)
def extractAmp(image, amp, corner, isTrimmed=False)
Definition: crosstalk.py:84
def writeCrosstalkCoeffs(outputFileName, coeff, det=None, crosstalkName="Unknown", indent=2)
Definition: crosstalk.py:202