lsst.cp.pipe  21.0.0-17-g5277551+2edf55539d
measureCrosstalk.py
Go to the documentation of this file.
1 # This file is part of cp_pipe
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import itertools
22 import numpy as np
23 
24 from collections import defaultdict
25 
26 import lsst.pipe.base as pipeBase
28 
29 from lsstDebug import getDebugFrame
30 from lsst.afw.detection import FootprintSet, Threshold
31 from lsst.afw.display import getDisplay
32 from lsst.pex.config import Config, Field, ListField, ConfigurableField
33 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
34 from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
35 from lsst.cp.pipe.utils import (ddict2dict, sigmaClipCorrection)
36 
37 from ._lookupStaticCalibration import lookupStaticCalibration
38 
39 __all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask",
40  "CrosstalkSolveTask", "CrosstalkSolveConfig",
41  "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
42 
43 
44 class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections,
45  dimensions=("instrument", "exposure", "detector")):
46  inputExp = cT.Input(
47  name="crosstalkInputs",
48  doc="Input post-ISR processed exposure to measure crosstalk from.",
49  storageClass="Exposure",
50  dimensions=("instrument", "exposure", "detector"),
51  multiple=False,
52  )
53  # TODO: Depends on DM-21904.
54  sourceExp = cT.Input(
55  name="crosstalkSource",
56  doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
57  storageClass="Exposure",
58  dimensions=("instrument", "exposure", "detector"),
59  multiple=True,
60  deferLoad=True,
61  # lookupFunction=None,
62  )
63 
64  outputRatios = cT.Output(
65  name="crosstalkRatios",
66  doc="Extracted crosstalk pixel ratios.",
67  storageClass="StructuredDataDict",
68  dimensions=("instrument", "exposure", "detector"),
69  )
70  outputFluxes = cT.Output(
71  name="crosstalkFluxes",
72  doc="Source pixel fluxes used in ratios.",
73  storageClass="StructuredDataDict",
74  dimensions=("instrument", "exposure", "detector"),
75  )
76 
77  def __init__(self, *, config=None):
78  super().__init__(config=config)
79  # Discard sourceExp until DM-21904 allows full interchip
80  # measurements.
81  self.inputs.discard("sourceExp")
82 
83 
84 class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig,
85  pipelineConnections=CrosstalkExtractConnections):
86  """Configuration for the measurement of pixel ratios.
87  """
88  doMeasureInterchip = Field(
89  dtype=bool,
90  default=False,
91  doc="Measure inter-chip crosstalk as well?",
92  )
93  threshold = Field(
94  dtype=float,
95  default=30000,
96  doc="Minimum level of source pixels for which to measure crosstalk."
97  )
98  ignoreSaturatedPixels = Field(
99  dtype=bool,
100  default=True,
101  doc="Should saturated pixels be ignored?"
102  )
103  badMask = ListField(
104  dtype=str,
105  default=["BAD", "INTRP"],
106  doc="Mask planes to ignore when identifying source pixels."
107  )
108  isTrimmed = Field(
109  dtype=bool,
110  default=True,
111  doc="Is the input exposure trimmed?"
112  )
113 
114  def validate(self):
115  super().validate()
116 
117  # Ensure the handling of the SAT mask plane is consistent
118  # with the ignoreSaturatedPixels value.
119  if self.ignoreSaturatedPixelsignoreSaturatedPixels:
120  if 'SAT' not in self.badMaskbadMask:
121  self.badMaskbadMask.append('SAT')
122  else:
123  if 'SAT' in self.badMaskbadMask:
124  self.badMaskbadMask = [mask for mask in self.badMaskbadMask if mask != 'SAT']
125 
126 
127 class CrosstalkExtractTask(pipeBase.PipelineTask,
128  pipeBase.CmdLineTask):
129  """Task to measure pixel ratios to find crosstalk.
130  """
131  ConfigClass = CrosstalkExtractConfig
132  _DefaultName = 'cpCrosstalkExtract'
133 
134  def run(self, inputExp, sourceExps=[]):
135  """Measure pixel ratios between amplifiers in inputExp.
136 
137  Extract crosstalk ratios between different amplifiers.
138 
139  For pixels above ``config.threshold``, we calculate the ratio
140  between each background-subtracted target amp and the source
141  amp. We return a list of ratios for each pixel for each
142  target/source combination, as nested dictionary containing the
143  ratio.
144 
145  Parameters
146  ----------
147  inputExp : `lsst.afw.image.Exposure`
148  Input exposure to measure pixel ratios on.
149  sourceExp : `list` [`lsst.afw.image.Exposure`], optional
150  List of chips to use as sources to measure inter-chip
151  crosstalk.
152 
153  Returns
154  -------
155  results : `lsst.pipe.base.Struct`
156  The results struct containing:
157 
158  ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
159  A catalog of ratio lists. The dictionaries are
160  indexed such that:
161  outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
162  contains the ratio list for that combination.
163  ``outputFluxes`` : `dict` [`dict` [`list`]]
164  A catalog of flux lists. The dictionaries are
165  indexed such that:
166  outputFluxes[sourceChip][sourceAmp]
167  contains the flux list used in the outputRatios.
168 
169  Notes
170  -----
171  The lsstDebug.Info() method can be rewritten for __name__ =
172  `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
173 
174  debug.display['extract'] : `bool`
175  Display the exposure under consideration, with the pixels used
176  for crosstalk measurement indicated by the DETECTED mask plane.
177  debug.display['pixels'] : `bool`
178  Display a plot of the ratio calculated for each pixel used in this
179  exposure, split by amplifier pairs. The median value is listed
180  for reference.
181  """
182  outputRatios = defaultdict(lambda: defaultdict(dict))
183  outputFluxes = defaultdict(lambda: defaultdict(dict))
184 
185  threshold = self.config.threshold
186  badPixels = list(self.config.badMask)
187 
188  targetDetector = inputExp.getDetector()
189  targetChip = targetDetector.getName()
190 
191  # Always look at the target chip first, then go to any other supplied exposures.
192  sourceExtractExps = [inputExp]
193  sourceExtractExps.extend(sourceExps)
194 
195  self.log.info("Measuring full detector background for target: %s", targetChip)
196  targetIm = inputExp.getMaskedImage()
197  FootprintSet(targetIm, Threshold(threshold), "DETECTED")
198  detected = targetIm.getMask().getPlaneBitMask("DETECTED")
199  bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
200 
201  self.debugViewdebugView('extract', inputExp)
202 
203  for sourceExp in sourceExtractExps:
204  sourceDetector = sourceExp.getDetector()
205  sourceChip = sourceDetector.getName()
206  sourceIm = sourceExp.getMaskedImage()
207  bad = sourceIm.getMask().getPlaneBitMask(badPixels)
208  self.log.info("Measuring crosstalk from source: %s", sourceChip)
209 
210  if sourceExp != inputExp:
211  FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
212  detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
213 
214  # The dictionary of amp-to-amp ratios for this pair of source->target detectors.
215  ratioDict = defaultdict(lambda: defaultdict(list))
216  extractedCount = 0
217 
218  for sourceAmp in sourceDetector:
219  sourceAmpName = sourceAmp.getName()
220  sourceAmpImage = sourceIm[sourceAmp.getBBox()]
221  sourceMask = sourceAmpImage.mask.array
222  select = ((sourceMask & detected > 0)
223  & (sourceMask & bad == 0)
224  & np.isfinite(sourceAmpImage.image.array))
225  count = np.sum(select)
226  self.log.debug(" Source amplifier: %s", sourceAmpName)
227 
228  outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
229 
230  for targetAmp in targetDetector:
231  # iterate over targetExposure
232  targetAmpName = targetAmp.getName()
233  if sourceAmpName == targetAmpName and sourceChip == targetChip:
234  ratioDict[sourceAmpName][targetAmpName] = []
235  continue
236  self.log.debug(" Target amplifier: %s", targetAmpName)
237 
238  targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
239  targetAmp, sourceAmp,
240  isTrimmed=self.config.isTrimmed)
241  ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
242  ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
243  extractedCount += count
244 
245  self.debugPixelsdebugPixels('pixels',
246  sourceAmpImage.image.array[select],
247  targetAmpImage.array[select] - bg,
248  sourceAmpName, targetAmpName)
249 
250  self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
251  extractedCount, sourceChip, targetChip, bg)
252  outputRatios[targetChip][sourceChip] = ratioDict
253 
254  return pipeBase.Struct(
255  outputRatios=ddict2dict(outputRatios),
256  outputFluxes=ddict2dict(outputFluxes)
257  )
258 
259  def debugView(self, stepname, exposure):
260  """Utility function to examine the image being processed.
261 
262  Parameters
263  ----------
264  stepname : `str`
265  State of processing to view.
266  exposure : `lsst.afw.image.Exposure`
267  Exposure to view.
268  """
269  frame = getDebugFrame(self._display, stepname)
270  if frame:
271  display = getDisplay(frame)
272  display.scale('asinh', 'zscale')
273  display.mtv(exposure)
274 
275  prompt = "Press Enter to continue: "
276  while True:
277  ans = input(prompt).lower()
278  if ans in ("", "c",):
279  break
280 
281  def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
282  """Utility function to examine the CT ratio pixel values.
283 
284  Parameters
285  ----------
286  stepname : `str`
287  State of processing to view.
288  pixelsIn : `np.ndarray`
289  Pixel values from the potential crosstalk source.
290  pixelsOut : `np.ndarray`
291  Pixel values from the potential crosstalk target.
292  sourceName : `str`
293  Source amplifier name
294  targetName : `str`
295  Target amplifier name
296  """
297  frame = getDebugFrame(self._display, stepname)
298  if frame:
299  import matplotlib.pyplot as plt
300  figure = plt.figure(1)
301  figure.clear()
302 
303  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
304  axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+')
305  plt.xlabel("Source amplifier pixel value")
306  plt.ylabel("Measured pixel ratio")
307  plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: "
308  f"{(np.median(pixelsOut / pixelsIn))}")
309  figure.show()
310 
311  prompt = "Press Enter to continue: "
312  while True:
313  ans = input(prompt).lower()
314  if ans in ("", "c",):
315  break
316  plt.close()
317 
318 
319 class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections,
320  dimensions=("instrument", "detector")):
321  inputRatios = cT.Input(
322  name="crosstalkRatios",
323  doc="Ratios measured for an input exposure.",
324  storageClass="StructuredDataDict",
325  dimensions=("instrument", "exposure", "detector"),
326  multiple=True,
327  )
328  inputFluxes = cT.Input(
329  name="crosstalkFluxes",
330  doc="Fluxes of CT source pixels, for nonlinear fits.",
331  storageClass="StructuredDataDict",
332  dimensions=("instrument", "exposure", "detector"),
333  multiple=True,
334  )
335  camera = cT.PrerequisiteInput(
336  name="camera",
337  doc="Camera the input data comes from.",
338  storageClass="Camera",
339  dimensions=("instrument",),
340  isCalibration=True,
341  lookupFunction=lookupStaticCalibration,
342  )
343 
344  outputCrosstalk = cT.Output(
345  name="crosstalk",
346  doc="Output proposed crosstalk calibration.",
347  storageClass="CrosstalkCalib",
348  dimensions=("instrument", "detector"),
349  multiple=False,
350  isCalibration=True,
351  )
352 
353  def __init__(self, *, config=None):
354  super().__init__(config=config)
355 
356  if config.fluxOrder == 0:
357  self.inputs.discard("inputFluxes")
358 
359 
360 class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig,
361  pipelineConnections=CrosstalkSolveConnections):
362  """Configuration for the solving of crosstalk from pixel ratios.
363  """
364  rejIter = Field(
365  dtype=int,
366  default=3,
367  doc="Number of rejection iterations for final coefficient calculation.",
368  )
369  rejSigma = Field(
370  dtype=float,
371  default=2.0,
372  doc="Rejection threshold (sigma) for final coefficient calculation.",
373  )
374  fluxOrder = Field(
375  dtype=int,
376  default=0,
377  doc="Polynomial order in source flux to fit crosstalk.",
378  )
379  doFiltering = Field(
380  dtype=bool,
381  default=False,
382  doc="Filter generated crosstalk to remove marginal measurements.",
383  )
384 
385 
386 class CrosstalkSolveTask(pipeBase.PipelineTask,
387  pipeBase.CmdLineTask):
388  """Task to solve crosstalk from pixel ratios.
389  """
390  ConfigClass = CrosstalkSolveConfig
391  _DefaultName = 'cpCrosstalkSolve'
392 
393  def runQuantum(self, butlerQC, inputRefs, outputRefs):
394  """Ensure that the input and output dimensions are passed along.
395 
396  Parameters
397  ----------
398  butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
399  Butler to operate on.
400  inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
401  Input data refs to load.
402  ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
403  Output data refs to persist.
404  """
405  inputs = butlerQC.get(inputRefs)
406 
407  # Use the dimensions to set calib/provenance information.
408  inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios]
409  inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
410 
411  outputs = self.runrun(**inputs)
412  butlerQC.put(outputs, outputRefs)
413 
414  def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
415  """Combine ratios to produce crosstalk coefficients.
416 
417  Parameters
418  ----------
419  inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
420  A list of nested dictionaries of ratios indexed by target
421  and source chip, then by target and source amplifier.
422  inputFluxes : `list` [`dict` [`dict` [`list`]]]
423  A list of nested dictionaries of source pixel fluxes, indexed
424  by source chip and amplifier.
425  camera : `lsst.afw.cameraGeom.Camera`
426  Input camera.
427  inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
428  DataIds to use to construct provenance.
429  outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
430  DataIds to use to populate the output calibration.
431 
432  Returns
433  -------
434  results : `lsst.pipe.base.Struct`
435  The results struct containing:
436 
437  ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
438  Final crosstalk calibration.
439  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
440  Provenance data for the new calibration.
441 
442  Raises
443  ------
444  RuntimeError
445  Raised if the input data contains multiple target detectors.
446 
447  Notes
448  -----
449  The lsstDebug.Info() method can be rewritten for __name__ =
450  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
451 
452  debug.display['reduce'] : `bool`
453  Display a histogram of the combined ratio measurements for
454  a pair of source/target amplifiers from all input
455  exposures/detectors.
456 
457  """
458  if outputDims:
459  calibChip = outputDims['detector']
460  instrument = outputDims['instrument']
461  else:
462  # calibChip needs to be set manually in Gen2.
463  calibChip = None
464  instrument = None
465 
466  self.log.info("Combining measurements from %d ratios and %d fluxes",
467  len(inputRatios), len(inputFluxes) if inputFluxes else 0)
468 
469  if inputFluxes is None:
470  inputFluxes = [None for exp in inputRatios]
471 
472  combinedRatios = defaultdict(lambda: defaultdict(list))
473  combinedFluxes = defaultdict(lambda: defaultdict(list))
474  for ratioDict, fluxDict in zip(inputRatios, inputFluxes):
475  for targetChip in ratioDict:
476  if calibChip and targetChip != calibChip:
477  raise RuntimeError("Received multiple target chips!")
478 
479  sourceChip = targetChip
480  if sourceChip in ratioDict[targetChip]:
481  ratios = ratioDict[targetChip][sourceChip]
482 
483  for targetAmp in ratios:
484  for sourceAmp in ratios[targetAmp]:
485  combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
486  if fluxDict:
487  combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
488  # TODO: DM-21904
489  # Iterating over all other entries in ratioDict[targetChip] will yield
490  # inter-chip terms.
491 
492  for targetAmp in combinedRatios:
493  for sourceAmp in combinedRatios[targetAmp]:
494  self.log.info("Read %d pixels for %s -> %s",
495  len(combinedRatios[targetAmp][sourceAmp]),
496  targetAmp, sourceAmp)
497  if len(combinedRatios[targetAmp][sourceAmp]) > 1:
498  self.debugRatiosdebugRatios('reduce', combinedRatios, targetAmp, sourceAmp)
499 
500  if self.config.fluxOrder == 0:
501  self.log.info("Fitting crosstalk coefficients.")
502  calib = self.measureCrosstalkCoefficientsmeasureCrosstalkCoefficients(combinedRatios,
503  self.config.rejIter, self.config.rejSigma)
504  else:
505  raise NotImplementedError("Non-linear crosstalk terms are not yet supported.")
506 
507  self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid))
508 
509  if self.config.doFiltering:
510  # This step will apply the calculated validity values to
511  # censor poorly measured coefficients.
512  self.log.info("Filtering measured crosstalk to remove invalid solutions.")
513  calib = self.filterCrosstalkCalibfilterCrosstalkCalib(calib)
514 
515  # Populate the remainder of the calibration information.
516  calib.hasCrosstalk = True
517  calib.interChip = {}
518 
519  # calibChip is the detector dimension, which is the detector Id
520  calib._detectorId = calibChip
521  if camera:
522  calib._detectorName = camera[calibChip].getName()
523  calib._detectorSerial = camera[calibChip].getSerial()
524 
525  calib._instrument = instrument
526  calib.updateMetadata(setCalibId=True, setDate=True)
527 
528  # Make an IsrProvenance().
529  provenance = IsrProvenance(calibType="CROSSTALK")
530  provenance._detectorName = calibChip
531  if inputDims:
532  provenance.fromDataIds(inputDims)
533  provenance._instrument = instrument
534  provenance.updateMetadata()
535 
536  return pipeBase.Struct(
537  outputCrosstalk=calib,
538  outputProvenance=provenance,
539  )
540 
541  def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma):
542  """Measure crosstalk coefficients from the ratios.
543 
544  Given a list of ratios for each target/source amp combination,
545  we measure a sigma clipped mean and error.
546 
547  The coefficient errors returned are the standard deviation of
548  the final set of clipped input ratios.
549 
550  Parameters
551  ----------
552  ratios : `dict` of `dict` of `numpy.ndarray`
553  Catalog of arrays of ratios.
554  rejIter : `int`
555  Number of rejection iterations.
556  rejSigma : `float`
557  Rejection threshold (sigma).
558 
559  Returns
560  -------
561  calib : `lsst.ip.isr.CrosstalkCalib`
562  The output crosstalk calibration.
563 
564  Notes
565  -----
566  The lsstDebug.Info() method can be rewritten for __name__ =
567  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
568 
569  debug.display['measure'] : `bool`
570  Display the CDF of the combined ratio measurements for
571  a pair of source/target amplifiers from the final set of
572  clipped input ratios.
573  """
574  calib = CrosstalkCalib(nAmp=len(ratios))
575 
576  # Calibration stores coefficients as a numpy ndarray.
577  ordering = list(ratios.keys())
578  for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
579  if ii == jj:
580  values = [0.0]
581  else:
582  values = np.array(ratios[ordering[ii]][ordering[jj]])
583  values = values[np.abs(values) < 1.0] # Discard unreasonable values
584 
585  calib.coeffNum[ii][jj] = len(values)
586 
587  if len(values) == 0:
588  self.log.warn("No values for matrix element %d,%d" % (ii, jj))
589  calib.coeffs[ii][jj] = np.nan
590  calib.coeffErr[ii][jj] = np.nan
591  calib.coeffValid[ii][jj] = False
592  else:
593  if ii != jj:
594  for rej in range(rejIter):
595  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
596  sigma = 0.741*(hi - lo)
597  good = np.abs(values - med) < rejSigma*sigma
598  if good.sum() == len(good):
599  break
600  values = values[good]
601 
602  calib.coeffs[ii][jj] = np.mean(values)
603  if calib.coeffNum[ii][jj] == 1:
604  calib.coeffErr[ii][jj] = np.nan
605  else:
606  correctionFactor = sigmaClipCorrection(rejSigma)
607  calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
608  calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj])
609  > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
610 
611  if calib.coeffNum[ii][jj] > 1:
612  self.debugRatiosdebugRatios('measure', ratios, ordering[ii], ordering[jj],
613  calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
614 
615  return calib
616 
617  @staticmethod
618  def filterCrosstalkCalib(inCalib):
619  """Apply valid constraints to the measured values.
620 
621  Any measured coefficient that is determined to be invalid is
622  set to zero, and has the error set to nan. The validation is
623  determined by checking that the measured coefficient is larger
624  than the calculated standard error of the mean.
625 
626  Parameters
627  ----------
628  inCalib : `lsst.ip.isr.CrosstalkCalib`
629  Input calibration to filter.
630 
631  Returns
632  -------
633  outCalib : `lsst.ip.isr.CrosstalkCalib`
634  Filtered calibration.
635  """
636  outCalib = CrosstalkCalib()
637  outCalib.numAmps = inCalib.numAmps
638 
639  outCalib.coeffs = inCalib.coeffs
640  outCalib.coeffs[~inCalib.coeffValid] = 0.0
641 
642  outCalib.coeffErr = inCalib.coeffErr
643  outCalib.coeffErr[~inCalib.coeffValid] = np.nan
644 
645  outCalib.coeffNum = inCalib.coeffNum
646  outCalib.coeffValid = inCalib.coeffValid
647 
648  return outCalib
649 
650  def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
651  """Utility function to examine the final CT ratio set.
652 
653  Parameters
654  ----------
655  stepname : `str`
656  State of processing to view.
657  ratios : `dict` of `dict` of `np.ndarray`
658  Array of measured CT ratios, indexed by source/victim
659  amplifier.
660  i : `str`
661  Index of the source amplifier.
662  j : `str`
663  Index of the target amplifier.
664  coeff : `float`, optional
665  Coefficient calculated to plot along with the simple mean.
666  valid : `bool`, optional
667  Validity to be added to the plot title.
668  """
669  frame = getDebugFrame(self._display, stepname)
670  if frame:
671  if i == j or ratios is None or len(ratios) < 1:
672  pass
673 
674  ratioList = ratios[i][j]
675  if ratioList is None or len(ratioList) < 1:
676  pass
677 
678  mean = np.mean(ratioList)
679  std = np.std(ratioList)
680  import matplotlib.pyplot as plt
681  figure = plt.figure(1)
682  figure.clear()
683  plt.hist(x=ratioList, bins=len(ratioList),
684  cumulative=True, color='b', density=True, histtype='step')
685  plt.xlabel("Measured pixel ratio")
686  plt.ylabel(f"CDF: n={len(ratioList)}")
687  plt.xlim(np.percentile(ratioList, [1.0, 99]))
688  plt.axvline(x=mean, color="k")
689  plt.axvline(x=coeff, color='g')
690  plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r')
691  plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r')
692  plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
693  figure.show()
694 
695  prompt = "Press Enter to continue: "
696  while True:
697  ans = input(prompt).lower()
698  if ans in ("", "c",):
699  break
700  elif ans in ("pdb", "p",):
701  import pdb
702  pdb.set_trace()
703  plt.close()
704 
705 
707  extract = ConfigurableField(
708  target=CrosstalkExtractTask,
709  doc="Task to measure pixel ratios.",
710  )
711  solver = ConfigurableField(
712  target=CrosstalkSolveTask,
713  doc="Task to convert ratio lists to crosstalk coefficients.",
714  )
715 
716 
717 class MeasureCrosstalkTask(pipeBase.CmdLineTask):
718  """Measure intra-detector crosstalk.
719 
720  See also
721  --------
722  lsst.ip.isr.crosstalk.CrosstalkCalib
723  lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
724  lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
725 
726  Notes
727  -----
728  The crosstalk this method measures assumes that when a bright
729  pixel is found in one detector amplifier, all other detector
730  amplifiers may see a signal change in the same pixel location
731  (relative to the readout amplifier) as these other pixels are read
732  out at the same time.
733 
734  After processing each input exposure through a limited set of ISR
735  stages, bright unmasked pixels above the threshold are identified.
736  The potential CT signal is found by taking the ratio of the
737  appropriate background-subtracted pixel value on the other
738  amplifiers to the input value on the source amplifier. If the
739  source amplifier has a large number of bright pixels as well, the
740  background level may be elevated, leading to poor ratio
741  measurements.
742 
743  The set of ratios found between each pair of amplifiers across all
744  input exposures is then gathered to produce the final CT
745  coefficients. The sigma-clipped mean and sigma are returned from
746  these sets of ratios, with the coefficient to supply to the ISR
747  CrosstalkTask() being the multiplicative inverse of these values.
748 
749  This Task simply calls the pipetask versions of the measure
750  crosstalk code.
751  """
752  ConfigClass = MeasureCrosstalkConfig
753  _DefaultName = "measureCrosstalk"
754 
755  # Let's use this instead of messing with parseAndRun.
756  RunnerClass = DataRefListRunner
757 
758  def __init__(self, **kwargs):
759  super().__init__(**kwargs)
760  self.makeSubtask("extract")
761  self.makeSubtask("solver")
762 
763  def runDataRef(self, dataRefList):
764  """Run extract task on each of inputs in the dataRef list, then pass
765  that to the solver task.
766 
767  Parameters
768  ----------
769  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
770  Data references for exposures for detectors to process.
771 
772  Returns
773  -------
774  results : `lsst.pipe.base.Struct`
775  The results struct containing:
776 
777  ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
778  Final crosstalk calibration.
779  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
780  Provenance data for the new calibration.
781 
782  Raises
783  ------
784  RuntimeError
785  Raised if multiple target detectors are supplied.
786  """
787  dataRef = dataRefList[0]
788  camera = dataRef.get("camera")
789 
790  ratios = []
791  activeChip = None
792  for dataRef in dataRefList:
793  exposure = dataRef.get("postISRCCD")
794  if activeChip:
795  if exposure.getDetector().getName() != activeChip:
796  raise RuntimeError("Too many input detectors supplied!")
797  else:
798  activeChip = exposure.getDetector().getName()
799 
800  self.extract.debugView("extract", exposure)
801  result = self.extract.run(exposure)
802  ratios.append(result.outputRatios)
803 
804  for detIter, detector in enumerate(camera):
805  if detector.getName() == activeChip:
806  detectorId = detIter
807  outputDims = {'instrument': camera.getName(),
808  'detector': detectorId,
809  }
810 
811  finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims)
812  dataRef.put(finalResults.outputCrosstalk, "crosstalk")
813 
814  return finalResults
def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName)
def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None)
def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False)
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def sigmaClipCorrection(nSigClip)
Definition: utils.py:42
def ddict2dict(d)
Definition: utils.py:749