lsst.cp.pipe  20.0.0-15-g4cba61e+2185b9856c
measureCrosstalk.py
Go to the documentation of this file.
1 # This file is part of cp_pipe
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import itertools
22 import numpy as np
23 from scipy.stats import norm
24 
25 from collections import defaultdict
26 
27 import lsst.pipe.base as pipeBase
29 
30 from lsstDebug import getDebugFrame
31 from lsst.afw.detection import FootprintSet, Threshold
32 from lsst.afw.display import getDisplay
33 from lsst.pex.config import Config, Field, ListField, ConfigurableField
34 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
35 from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
36 
37 __all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask",
38  "CrosstalkSolveTask", "CrosstalkSolveConfig",
39  "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
40 
41 
42 class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections,
43  dimensions=("instrument", "exposure", "detector")):
44  inputExp = cT.Input(
45  name="crosstalkInputs",
46  doc="Input post-ISR processed exposure to measure crosstalk from.",
47  storageClass="Exposure",
48  dimensions=("instrument", "exposure", "detector"),
49  multiple=False,
50  )
51  # TODO: Depends on DM-21904.
52  sourceExp = cT.Input(
53  name="crosstalkSource",
54  doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
55  storageClass="Exposure",
56  dimensions=("instrument", "exposure", "detector"),
57  multiple=True,
58  deferLoad=True,
59  # lookupFunction=None,
60  )
61 
62  outputRatios = cT.Output(
63  name="crosstalkRatios",
64  doc="Extracted crosstalk pixel ratios.",
65  storageClass="StructuredDataDict",
66  dimensions=("instrument", "exposure", "detector"),
67  )
68  outputFluxes = cT.Output(
69  name="crosstalkFluxes",
70  doc="Source pixel fluxes used in ratios.",
71  storageClass="StructuredDataDict",
72  dimensions=("instrument", "exposure", "detector"),
73  )
74 
75  def __init__(self, *, config=None):
76  super().__init__(config=config)
77  # Discard sourceExp until DM-21904 allows full interchip
78  # measurements.
79  self.inputs.discard("sourceExp")
80 
81 
82 class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig,
83  pipelineConnections=CrosstalkExtractConnections):
84  """Configuration for the measurement of pixel ratios.
85  """
86  doMeasureInterchip = Field(
87  dtype=bool,
88  default=False,
89  doc="Measure inter-chip crosstalk as well?",
90  )
91  threshold = Field(
92  dtype=float,
93  default=30000,
94  doc="Minimum level of source pixels for which to measure crosstalk."
95  )
96  ignoreSaturatedPixels = Field(
97  dtype=bool,
98  default=True,
99  doc="Should saturated pixels be ignored?"
100  )
101  badMask = ListField(
102  dtype=str,
103  default=["BAD", "INTRP"],
104  doc="Mask planes to ignore when identifying source pixels."
105  )
106  isTrimmed = Field(
107  dtype=bool,
108  default=True,
109  doc="Is the input exposure trimmed?"
110  )
111 
112  def validate(self):
113  super().validate()
114 
115  # Ensure the handling of the SAT mask plane is consistent
116  # with the ignoreSaturatedPixels value.
117  if self.ignoreSaturatedPixels:
118  if 'SAT' not in self.badMask:
119  self.badMask.append('SAT')
120  else:
121  if 'SAT' in self.badMask:
122  self.badMask = [mask for mask in self.badMask if mask != 'SAT']
123 
124 
125 class CrosstalkExtractTask(pipeBase.PipelineTask,
126  pipeBase.CmdLineTask):
127  """Task to measure pixel ratios to find crosstalk.
128  """
129  ConfigClass = CrosstalkExtractConfig
130  _DefaultName = 'cpCrosstalkExtract'
131 
132  def run(self, inputExp, sourceExps=[]):
133  """Measure pixel ratios between amplifiers in inputExp.
134 
135  Extract crosstalk ratios between different amplifiers.
136 
137  For pixels above ``config.threshold``, we calculate the ratio
138  between each background-subtracted target amp and the source
139  amp. We return a list of ratios for each pixel for each
140  target/source combination, as nested dictionary containing the
141  ratio.
142 
143  Parameters
144  ----------
145  inputExp : `lsst.afw.image.Exposure`
146  Input exposure to measure pixel ratios on.
147  sourceExp : `list` [`lsst.afw.image.Exposure`], optional
148  List of chips to use as sources to measure inter-chip
149  crosstalk.
150 
151  Returns
152  -------
153  results : `lsst.pipe.base.Struct`
154  The results struct containing:
155 
156  ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
157  A catalog of ratio lists. The dictionaries are
158  indexed such that:
159  outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
160  contains the ratio list for that combination.
161  ``outputFluxes`` : `dict` [`dict` [`list`]]
162  A catalog of flux lists. The dictionaries are
163  indexed such that:
164  outputFluxes[sourceChip][sourceAmp]
165  contains the flux list used in the outputRatios.
166 
167  Notes
168  -----
169  The lsstDebug.Info() method can be rewritten for __name__ =
170  `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
171 
172  debug.display['extract'] : `bool`
173  Display the exposure under consideration, with the pixels used
174  for crosstalk measurement indicated by the DETECTED mask plane.
175  debug.display['pixels'] : `bool`
176  Display a plot of the ratio calculated for each pixel used in this
177  exposure, split by amplifier pairs. The median value is listed
178  for reference.
179  """
180  outputRatios = defaultdict(lambda: defaultdict(dict))
181  outputFluxes = defaultdict(lambda: defaultdict(dict))
182 
183  threshold = self.config.threshold
184  badPixels = list(self.config.badMask)
185 
186  targetDetector = inputExp.getDetector()
187  targetChip = targetDetector.getName()
188 
189  # Always look at the target chip first, then go to any other supplied exposures.
190  sourceExtractExps = [inputExp]
191  sourceExtractExps.extend(sourceExps)
192 
193  self.log.info("Measuring full detector background for target: %s", targetChip)
194  targetIm = inputExp.getMaskedImage()
195  FootprintSet(targetIm, Threshold(threshold), "DETECTED")
196  detected = targetIm.getMask().getPlaneBitMask("DETECTED")
197  bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
198 
199  self.debugView('extract', inputExp)
200 
201  for sourceExp in sourceExtractExps:
202  sourceDetector = sourceExp.getDetector()
203  sourceChip = sourceDetector.getName()
204  sourceIm = sourceExp.getMaskedImage()
205  bad = sourceIm.getMask().getPlaneBitMask(badPixels)
206  self.log.info("Measuring crosstalk from source: %s", sourceChip)
207 
208  if sourceExp != inputExp:
209  FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
210  detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
211 
212  # The dictionary of amp-to-amp ratios for this pair of source->target detectors.
213  ratioDict = defaultdict(lambda: defaultdict(list))
214  extractedCount = 0
215 
216  for sourceAmp in sourceDetector:
217  sourceAmpName = sourceAmp.getName()
218  sourceAmpImage = sourceIm[sourceAmp.getBBox()]
219  sourceMask = sourceAmpImage.mask.array
220  select = ((sourceMask & detected > 0) &
221  (sourceMask & bad == 0) &
222  np.isfinite(sourceAmpImage.image.array))
223  count = np.sum(select)
224  self.log.debug(" Source amplifier: %s", sourceAmpName)
225 
226  outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select]
227 
228  for targetAmp in targetDetector:
229  # iterate over targetExposure
230  targetAmpName = targetAmp.getName()
231  if sourceAmpName == targetAmpName and sourceChip == targetChip:
232  ratioDict[sourceAmpName][targetAmpName] = []
233  continue
234  self.log.debug(" Target amplifier: %s", targetAmpName)
235 
236  targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
237  targetAmp, sourceAmp,
238  isTrimmed=self.config.isTrimmed)
239  ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
240  ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
241  extractedCount += count
242 
243  self.debugPixels('pixels',
244  sourceAmpImage.image.array[select],
245  targetAmpImage.array[select] - bg,
246  sourceAmpName, targetAmpName)
247 
248  self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
249  extractedCount, sourceChip, targetChip, bg)
250  outputRatios[targetChip][sourceChip] = ratioDict
251 
252  return pipeBase.Struct(
253  outputRatios=outputRatios,
254  outputFluxes=outputFluxes
255  )
256 
257  def debugView(self, stepname, exposure):
258  """Utility function to examine the image being processed.
259 
260  Parameters
261  ----------
262  stepname : `str`
263  State of processing to view.
264  exposure : `lsst.afw.image.Exposure`
265  Exposure to view.
266  """
267  frame = getDebugFrame(self._display, stepname)
268  if frame:
269  display = getDisplay(frame)
270  display.scale('asinh', 'zscale')
271  display.mtv(exposure)
272 
273  prompt = "Press Enter to continue: "
274  while True:
275  ans = input(prompt).lower()
276  if ans in ("", "c",):
277  break
278 
279  def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
280  """Utility function to examine the CT ratio pixel values.
281 
282  Parameters
283  ----------
284  stepname : `str`
285  State of processing to view.
286  pixelsIn : `np.ndarray`
287  Pixel values from the potential crosstalk source.
288  pixelsOut : `np.ndarray`
289  Pixel values from the potential crosstalk target.
290  sourceName : `str`
291  Source amplifier name
292  targetName : `str`
293  Target amplifier name
294  """
295  frame = getDebugFrame(self._display, stepname)
296  if frame:
297  import matplotlib.pyplot as plt
298  figure = plt.figure(1)
299  figure.clear()
300 
301  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
302  axes.plt(pixelsIn, pixelsOut / pixelsIn, 'k+')
303  plt.xlabel("Source amplifier pixel value")
304  plt.ylabel("Measured pixel ratio")
305  plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: "
306  f"{(np.median(pixelsOut / pixelsIn))}")
307  figure.show()
308 
309  prompt = "Press Enter to continue: "
310  while True:
311  ans = input(prompt).lower()
312  if ans in ("", "c",):
313  break
314  plt.close()
315 
316 
317 class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections,
318  dimensions=("instrument", "detector")):
319  inputRatios = cT.Input(
320  name="crosstalkRatios",
321  doc="Ratios measured for an input exposure.",
322  storageClass="StructuredDataDict",
323  dimensions=("instrument", "exposure", "detector"),
324  multiple=True,
325  )
326  inputFluxes = cT.Input(
327  name="crosstalkFluxes",
328  doc="Fluxes of CT source pixels, for nonlinear fits.",
329  storageClass="StructuredDataDict",
330  dimensions=("instrument", "exposure", "detector"),
331  multiple=True,
332  )
333  camera = cT.PrerequisiteInput(
334  name="camera",
335  doc="Camera the input data comes from.",
336  storageClass="Camera",
337  dimensions=("instrument", "calibration_label"),
338  )
339 
340  outputCrosstalk = cT.Output(
341  name="crosstalkProposal",
342  doc="Output proposed crosstalk calibration.",
343  storageClass="CrosstalkCalib",
344  dimensions=("instrument", "detector"),
345  multiple=False,
346  )
347 
348  def __init__(self, *, config=None):
349  super().__init__(config=config)
350 
351  if config.fluxOrder == 0:
352  self.inputs.discard("inputFluxes")
353 
354 
355 class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig,
356  pipelineConnections=CrosstalkSolveConnections):
357  """Configuration for the solving of crosstalk from pixel ratios.
358  """
359  rejIter = Field(
360  dtype=int,
361  default=3,
362  doc="Number of rejection iterations for final coefficient calculation.",
363  )
364  rejSigma = Field(
365  dtype=float,
366  default=2.0,
367  doc="Rejection threshold (sigma) for final coefficient calculation.",
368  )
369  fluxOrder = Field(
370  dtype=int,
371  default=0,
372  doc="Polynomial order in source flux to fit crosstalk.",
373  )
374  doFiltering = Field(
375  dtype=bool,
376  default=False,
377  doc="Filter generated crosstalk to remove marginal measurements.",
378  )
379 
380 
381 class CrosstalkSolveTask(pipeBase.PipelineTask,
382  pipeBase.CmdLineTask):
383  """Task to solve crosstalk from pixel ratios.
384  """
385  ConfigClass = CrosstalkSolveConfig
386  _DefaultName = 'cpCrosstalkSolve'
387 
388  def runQuantum(self, butlerQC, inputRefs, outputRefs):
389  """Ensure that the input and output dimensions are passed along.
390 
391  Parameters
392  ----------
393  butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
394  Butler to operate on.
395  inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
396  Input data refs to load.
397  ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
398  Output data refs to persist.
399  """
400  inputs = butlerQC.get(inputRefs)
401 
402  # Use the dimensions to set calib/provenance information.
403  inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios]
404  inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
405 
406  outputs = self.run(**inputs)
407  butlerQC.put(outputs, outputRefs)
408 
409  def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
410  """Combine ratios to produce crosstalk coefficients.
411 
412  Parameters
413  ----------
414  inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
415  A list of nested dictionaries of ratios indexed by target
416  and source chip, then by target and source amplifier.
417  inputFluxes : `list` [`dict` [`dict` [`list`]]]
418  A list of nested dictionaries of source pixel fluxes, indexed
419  by source chip and amplifier.
420  camera : `lsst.afw.cameraGeom.Camera`
421  Input camera.
422  inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
423  DataIds to use to construct provenance.
424  outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
425  DataIds to use to populate the output calibration.
426 
427  Returns
428  -------
429  results : `lsst.pipe.base.Struct`
430  The results struct containing:
431 
432  ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
433  Final crosstalk calibration.
434  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
435  Provenance data for the new calibration.
436 
437  Raises
438  ------
439  RuntimeError
440  Raised if the input data contains multiple target detectors.
441 
442  Notes
443  -----
444  The lsstDebug.Info() method can be rewritten for __name__ =
445  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
446 
447  debug.display['reduce'] : `bool`
448  Display a histogram of the combined ratio measurements for
449  a pair of source/target amplifiers from all input
450  exposures/detectors.
451 
452  """
453  if outputDims:
454  calibChip = outputDims['detector']
455  instrument = outputDims['instrument']
456  else:
457  # calibChip needs to be set manually in Gen2.
458  calibChip = None
459  instrument = None
460 
461  self.log.info("Combining measurements from %d ratios and %d fluxes",
462  len(inputRatios), len(inputFluxes) if inputFluxes else 0)
463 
464  if inputFluxes is None:
465  inputFluxes = [None for exp in inputRatios]
466 
467  combinedRatios = defaultdict(lambda: defaultdict(list))
468  combinedFluxes = defaultdict(lambda: defaultdict(list))
469  for ratioDict, fluxDict in zip(inputRatios, inputFluxes):
470  for targetChip in ratioDict:
471  if calibChip and targetChip != calibChip:
472  raise RuntimeError("Received multiple target chips!")
473 
474  sourceChip = targetChip
475  if sourceChip in ratioDict[targetChip]:
476  ratios = ratioDict[targetChip][sourceChip]
477 
478  for targetAmp in ratios:
479  for sourceAmp in ratios[targetAmp]:
480  combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
481  if fluxDict:
482  combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
483  # TODO: DM-21904
484  # Iterating over all other entries in ratioDict[targetChip] will yield
485  # inter-chip terms.
486 
487  for targetAmp in combinedRatios:
488  for sourceAmp in combinedRatios[targetAmp]:
489  self.log.info("Read %d pixels for %s -> %s",
490  len(combinedRatios[targetAmp][sourceAmp]),
491  targetAmp, sourceAmp)
492  if len(combinedRatios[targetAmp][sourceAmp]) > 1:
493  self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp)
494 
495  if self.config.fluxOrder == 0:
496  self.log.info("Fitting crosstalk coefficients.")
497  calib = self.measureCrosstalkCoefficients(combinedRatios,
498  self.config.rejIter, self.config.rejSigma)
499  else:
500  raise NotImplementedError("Non-linear crosstalk terms are not yet supported.")
501 
502  self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid))
503 
504  if self.config.doFiltering:
505  # This step will apply the calculated validity values to
506  # censor poorly measured coefficients.
507  self.log.info("Filtering measured crosstalk to remove invalid solutions.")
508  calib = self.filterCrosstalkCalib(calib)
509 
510  # Populate the remainder of the calibration information.
511  calib.hasCrosstalk = True
512  calib.interChip = {}
513  calib._detectorName = calibChip
514  if camera:
515  for chip in camera:
516  if chip.getName() == calibChip:
517  calib._detectorSerial = chip.getSerial()
518  calib._instrument = instrument
519  calib.updateMetadata()
520 
521  # Make an IsrProvenance().
522  provenance = IsrProvenance(calibType="CROSSTALK")
523  provenance._detectorName = calibChip
524  if inputDims:
525  provenance.fromDataIds(inputDims)
526  provenance._instrument = instrument
527  provenance.updateMetadata()
528 
529  return pipeBase.Struct(
530  outputCrosstalk=calib,
531  outputProvenance=provenance,
532  )
533 
534  def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma):
535  """Measure crosstalk coefficients from the ratios.
536 
537  Given a list of ratios for each target/source amp combination,
538  we measure a sigma clipped mean and error.
539 
540  The coefficient errors returned are the standard deviation of
541  the final set of clipped input ratios.
542 
543  Parameters
544  ----------
545  ratios : `dict` of `dict` of `numpy.ndarray`
546  Catalog of arrays of ratios.
547  rejIter : `int`
548  Number of rejection iterations.
549  rejSigma : `float`
550  Rejection threshold (sigma).
551 
552  Returns
553  -------
554  calib : `lsst.ip.isr.CrosstalkCalib`
555  The output crosstalk calibration.
556 
557  Notes
558  -----
559  The lsstDebug.Info() method can be rewritten for __name__ =
560  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
561 
562  debug.display['measure'] : `bool`
563  Display the CDF of the combined ratio measurements for
564  a pair of source/target amplifiers from the final set of
565  clipped input ratios.
566  """
567  calib = CrosstalkCalib(nAmp=len(ratios))
568 
569  # Calibration stores coefficients as a numpy ndarray.
570  ordering = list(ratios.keys())
571  for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
572  if ii == jj:
573  values = [0.0]
574  else:
575  values = np.array(ratios[ordering[ii]][ordering[jj]])
576  values = values[np.abs(values) < 1.0] # Discard unreasonable values
577 
578  calib.coeffNum[ii][jj] = len(values)
579 
580  if len(values) == 0:
581  self.log.warn("No values for matrix element %d,%d" % (ii, jj))
582  calib.coeffs[ii][jj] = np.nan
583  calib.coeffErr[ii][jj] = np.nan
584  calib.coeffValid[ii][jj] = False
585  else:
586  if ii != jj:
587  for rej in range(rejIter):
588  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
589  sigma = 0.741*(hi - lo)
590  good = np.abs(values - med) < rejSigma*sigma
591  if good.sum() == len(good):
592  break
593  values = values[good]
594 
595  calib.coeffs[ii][jj] = np.mean(values)
596  if calib.coeffNum[ii][jj] == 1:
597  calib.coeffErr[ii][jj] = np.nan
598  else:
599  correctionFactor = self.sigmaClipCorrection(rejSigma)
600  calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
601  calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) >
602  calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
603 
604  if calib.coeffNum[ii][jj] > 1:
605  self.debugRatios('measure', ratios, ordering[ii], ordering[jj],
606  calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
607 
608  return calib
609 
610  @staticmethod
611  def sigmaClipCorrection(nSigClip):
612  """Correct measured sigma to account for clipping.
613 
614  If we clip our input data and then measure sigma, then the
615  measured sigma is smaller than the true value because real
616  points beyond the clip threshold have been removed. This is a
617  small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the
618  default parameters for measure crosstalk use nSigClip=2.0.
619  This causes the measured sigma to be about 15% smaller than
620  real. This formula corrects the issue, for the symmetric case
621  (upper clip threshold equal to lower clip threshold).
622 
623  Parameters
624  ----------
625  nSigClip : `float`
626  Number of sigma the measurement was clipped by.
627 
628  Returns
629  -------
630  scaleFactor : `float`
631  Scale factor to increase the measured sigma by.
632 
633  """
634  varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip))
635  return 1.0 / np.sqrt(varFactor)
636 
637  @staticmethod
638  def filterCrosstalkCalib(inCalib):
639  """Apply valid constraints to the measured values.
640 
641  Any measured coefficient that is determined to be invalid is
642  set to zero, and has the error set to nan. The validation is
643  determined by checking that the measured coefficient is larger
644  than the calculated standard error of the mean.
645 
646  Parameters
647  ----------
648  inCalib : `lsst.ip.isr.CrosstalkCalib`
649  Input calibration to filter.
650 
651  Returns
652  -------
653  outCalib : `lsst.ip.isr.CrosstalkCalib`
654  Filtered calibration.
655  """
656  outCalib = CrosstalkCalib()
657  outCalib.numAmps = inCalib.numAmps
658 
659  outCalib.coeffs = inCalib.coeffs
660  outCalib.coeffs[~inCalib.coeffValid] = 0.0
661 
662  outCalib.coeffErr = inCalib.coeffErr
663  outCalib.coeffErr[~inCalib.coeffValid] = np.nan
664 
665  outCalib.coeffNum = inCalib.coeffNum
666  outCalib.coeffValid = inCalib.coeffValid
667 
668  return outCalib
669 
670  def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
671  """Utility function to examine the final CT ratio set.
672 
673  Parameters
674  ----------
675  stepname : `str`
676  State of processing to view.
677  ratios : `dict` of `dict` of `np.ndarray`
678  Array of measured CT ratios, indexed by source/victim
679  amplifier.
680  i : `str`
681  Index of the source amplifier.
682  j : `str`
683  Index of the target amplifier.
684  coeff : `float`, optional
685  Coefficient calculated to plot along with the simple mean.
686  valid : `bool`, optional
687  Validity to be added to the plot title.
688  """
689  frame = getDebugFrame(self._display, stepname)
690  if frame:
691  if i == j or ratios is None or len(ratios) < 1:
692  pass
693 
694  ratioList = ratios[i][j]
695  if ratioList is None or len(ratioList) < 1:
696  pass
697 
698  mean = np.mean(ratioList)
699  std = np.std(ratioList)
700  import matplotlib.pyplot as plt
701  figure = plt.figure(1)
702  figure.clear()
703  plt.hist(x=ratioList, bins=len(ratioList),
704  cumulative=True, color='b', density=True, histtype='step')
705  plt.xlabel("Measured pixel ratio")
706  plt.ylabel(f"CDF: n={len(ratioList)}")
707  plt.xlim(np.percentile(ratioList, [1.0, 99]))
708  plt.axvline(x=mean, color="k")
709  plt.axvline(x=coeff, color='g')
710  plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r')
711  plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r')
712  plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
713  figure.show()
714 
715  prompt = "Press Enter to continue: "
716  while True:
717  ans = input(prompt).lower()
718  if ans in ("", "c",):
719  break
720  elif ans in ("pdb", "p",):
721  import pdb
722  pdb.set_trace()
723  plt.close()
724 
725 
727  extract = ConfigurableField(
728  target=CrosstalkExtractTask,
729  doc="Task to measure pixel ratios.",
730  )
731  solver = ConfigurableField(
732  target=CrosstalkSolveTask,
733  doc="Task to convert ratio lists to crosstalk coefficients.",
734  )
735 
736 
737 class MeasureCrosstalkTask(pipeBase.CmdLineTask):
738  """Measure intra-detector crosstalk.
739 
740  See also
741  --------
742  lsst.ip.isr.crosstalk.CrosstalkCalib
743  lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
744  lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
745 
746  Notes
747  -----
748  The crosstalk this method measures assumes that when a bright
749  pixel is found in one detector amplifier, all other detector
750  amplifiers may see a signal change in the same pixel location
751  (relative to the readout amplifier) as these other pixels are read
752  out at the same time.
753 
754  After processing each input exposure through a limited set of ISR
755  stages, bright unmasked pixels above the threshold are identified.
756  The potential CT signal is found by taking the ratio of the
757  appropriate background-subtracted pixel value on the other
758  amplifiers to the input value on the source amplifier. If the
759  source amplifier has a large number of bright pixels as well, the
760  background level may be elevated, leading to poor ratio
761  measurements.
762 
763  The set of ratios found between each pair of amplifiers across all
764  input exposures is then gathered to produce the final CT
765  coefficients. The sigma-clipped mean and sigma are returned from
766  these sets of ratios, with the coefficient to supply to the ISR
767  CrosstalkTask() being the multiplicative inverse of these values.
768 
769  This Task simply calls the pipetask versions of the measure
770  crosstalk code.
771  """
772  ConfigClass = MeasureCrosstalkConfig
773  _DefaultName = "measureCrosstalk"
774 
775  # Let's use this instead of messing with parseAndRun.
776  RunnerClass = DataRefListRunner
777 
778  def __init__(self, **kwargs):
779  super().__init__(**kwargs)
780  self.makeSubtask("extract")
781  self.makeSubtask("solver")
782 
783  def runDataRef(self, dataRefList):
784  """Run extract task on each of inputs in the dataRef list, then pass
785  that to the solver task.
786 
787  Parameters
788  ----------
789  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
790  Data references for exposures for detectors to process.
791 
792  Returns
793  -------
794  results : `lsst.pipe.base.Struct`
795  The results struct containing:
796 
797  ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
798  Final crosstalk calibration.
799  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
800  Provenance data for the new calibration.
801 
802  Raises
803  ------
804  RuntimeError
805  Raised if multiple target detectors are supplied.
806  """
807  dataRef = dataRefList[0]
808  camera = dataRef.get("camera")
809 
810  ratios = []
811  activeChip = None
812  for dataRef in dataRefList:
813  exposure = dataRef.get("postISRCCD")
814  if activeChip:
815  if exposure.getDetector().getName() != activeChip:
816  raise RuntimeError("Too many input detectors supplied!")
817  else:
818  activeChip = exposure.getDetector().getName()
819 
820  self.extract.debugView("extract", exposure)
821  result = self.extract.run(exposure)
822  ratios.append(result.outputRatios)
823 
824  finalResults = self.solver.run(ratios, camera=camera)
825  dataRef.put(finalResults.outputCrosstalk, "crosstalk")
826 
827  return finalResults
lsst.cp.pipe.measureCrosstalk.MeasureCrosstalkConfig
Definition: measureCrosstalk.py:726
lsst.cp.pipe.measureCrosstalk.MeasureCrosstalkTask.runDataRef
def runDataRef(self, dataRefList)
Definition: measureCrosstalk.py:783
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveConnections
Definition: measureCrosstalk.py:318
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveConfig
Definition: measureCrosstalk.py:356
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractConfig.badMask
badMask
Definition: measureCrosstalk.py:101
lsst.cp.pipe.measureCrosstalk.MeasureCrosstalkTask
Definition: measureCrosstalk.py:737
lsst::afw::display
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractConnections.__init__
def __init__(self, *config=None)
Definition: measureCrosstalk.py:75
lsst.cp.pipe.measureCrosstalk.MeasureCrosstalkTask.__init__
def __init__(self, **kwargs)
Definition: measureCrosstalk.py:778
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask.debugRatios
def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False)
Definition: measureCrosstalk.py:670
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveConnections.__init__
def __init__(self, *config=None)
Definition: measureCrosstalk.py:348
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractConfig
Definition: measureCrosstalk.py:83
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractConfig.ignoreSaturatedPixels
ignoreSaturatedPixels
Definition: measureCrosstalk.py:96
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
Definition: measureCrosstalk.py:382
lsst::afw::detection::FootprintSet
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
Definition: measureCrosstalk.py:126
lsst::afw::detection::Threshold
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask.sigmaClipCorrection
def sigmaClipCorrection(nSigClip)
Definition: measureCrosstalk.py:611
lsst::pex::config
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask.run
def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None)
Definition: measureCrosstalk.py:409
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask.run
def run(self, inputExp, sourceExps=[])
Definition: measureCrosstalk.py:132
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask.debugPixels
def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName)
Definition: measureCrosstalk.py:279
lsst::afw::detection
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractConnections
Definition: measureCrosstalk.py:43
lsst::ip::isr
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: measureCrosstalk.py:388
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractConfig.validate
def validate(self)
Definition: measureCrosstalk.py:112
lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask.debugView
def debugView(self, stepname, exposure)
Definition: measureCrosstalk.py:257
lsst::pipe::base
lsst::pipe::base::connectionTypes
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask.measureCrosstalkCoefficients
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma)
Definition: measureCrosstalk.py:534
lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask.filterCrosstalkCalib
def filterCrosstalkCalib(inCalib)
Definition: measureCrosstalk.py:638