lsst.cp.pipe  21.0.0-19-g0e82efe+d8cd6b0ac0
measureCrosstalk.py
Go to the documentation of this file.
1 # This file is part of cp_pipe
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import itertools
22 import numpy as np
23 
24 from collections import defaultdict
25 
26 import lsst.pipe.base as pipeBase
28 
29 from lsstDebug import getDebugFrame
30 from lsst.afw.detection import FootprintSet, Threshold
31 from lsst.afw.display import getDisplay
32 from lsst.pex.config import Config, Field, ListField, ConfigurableField
33 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
34 from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
35 from lsst.cp.pipe.utils import (ddict2dict, sigmaClipCorrection)
36 
37 from ._lookupStaticCalibration import lookupStaticCalibration
38 
39 __all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask",
40  "CrosstalkSolveTask", "CrosstalkSolveConfig",
41  "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
42 
43 
44 class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections,
45  dimensions=("instrument", "exposure", "detector")):
46  inputExp = cT.Input(
47  name="crosstalkInputs",
48  doc="Input post-ISR processed exposure to measure crosstalk from.",
49  storageClass="Exposure",
50  dimensions=("instrument", "exposure", "detector"),
51  multiple=False,
52  )
53  # TODO: Depends on DM-21904.
54  sourceExp = cT.Input(
55  name="crosstalkSource",
56  doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
57  storageClass="Exposure",
58  dimensions=("instrument", "exposure", "detector"),
59  multiple=True,
60  deferLoad=True,
61  # lookupFunction=None,
62  )
63 
64  outputRatios = cT.Output(
65  name="crosstalkRatios",
66  doc="Extracted crosstalk pixel ratios.",
67  storageClass="StructuredDataDict",
68  dimensions=("instrument", "exposure", "detector"),
69  )
70  outputFluxes = cT.Output(
71  name="crosstalkFluxes",
72  doc="Source pixel fluxes used in ratios.",
73  storageClass="StructuredDataDict",
74  dimensions=("instrument", "exposure", "detector"),
75  )
76 
77  def __init__(self, *, config=None):
78  super().__init__(config=config)
79  # Discard sourceExp until DM-21904 allows full interchip
80  # measurements.
81  self.inputs.discard("sourceExp")
82 
83 
84 class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig,
85  pipelineConnections=CrosstalkExtractConnections):
86  """Configuration for the measurement of pixel ratios.
87  """
88  doMeasureInterchip = Field(
89  dtype=bool,
90  default=False,
91  doc="Measure inter-chip crosstalk as well?",
92  )
93  threshold = Field(
94  dtype=float,
95  default=30000,
96  doc="Minimum level of source pixels for which to measure crosstalk."
97  )
98  ignoreSaturatedPixels = Field(
99  dtype=bool,
100  default=True,
101  doc="Should saturated pixels be ignored?"
102  )
103  badMask = ListField(
104  dtype=str,
105  default=["BAD", "INTRP"],
106  doc="Mask planes to ignore when identifying source pixels."
107  )
108  isTrimmed = Field(
109  dtype=bool,
110  default=True,
111  doc="Is the input exposure trimmed?"
112  )
113 
114  def validate(self):
115  super().validate()
116 
117  # Ensure the handling of the SAT mask plane is consistent
118  # with the ignoreSaturatedPixels value.
119  if self.ignoreSaturatedPixelsignoreSaturatedPixels:
120  if 'SAT' not in self.badMaskbadMask:
121  self.badMaskbadMask.append('SAT')
122  else:
123  if 'SAT' in self.badMaskbadMask:
124  self.badMaskbadMask = [mask for mask in self.badMaskbadMask if mask != 'SAT']
125 
126 
127 class CrosstalkExtractTask(pipeBase.PipelineTask,
128  pipeBase.CmdLineTask):
129  """Task to measure pixel ratios to find crosstalk.
130  """
131  ConfigClass = CrosstalkExtractConfig
132  _DefaultName = 'cpCrosstalkExtract'
133 
134  def run(self, inputExp, sourceExps=[]):
135  """Measure pixel ratios between amplifiers in inputExp.
136 
137  Extract crosstalk ratios between different amplifiers.
138 
139  For pixels above ``config.threshold``, we calculate the ratio
140  between each background-subtracted target amp and the source
141  amp. We return a list of ratios for each pixel for each
142  target/source combination, as nested dictionary containing the
143  ratio.
144 
145  Parameters
146  ----------
147  inputExp : `lsst.afw.image.Exposure`
148  Input exposure to measure pixel ratios on.
149  sourceExp : `list` [`lsst.afw.image.Exposure`], optional
150  List of chips to use as sources to measure inter-chip
151  crosstalk.
152 
153  Returns
154  -------
155  results : `lsst.pipe.base.Struct`
156  The results struct containing:
157 
158  ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
159  A catalog of ratio lists. The dictionaries are
160  indexed such that:
161  outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
162  contains the ratio list for that combination.
163  ``outputFluxes`` : `dict` [`dict` [`list`]]
164  A catalog of flux lists. The dictionaries are
165  indexed such that:
166  outputFluxes[sourceChip][sourceAmp]
167  contains the flux list used in the outputRatios.
168 
169  Notes
170  -----
171  The lsstDebug.Info() method can be rewritten for __name__ =
172  `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
173 
174  debug.display['extract'] : `bool`
175  Display the exposure under consideration, with the pixels used
176  for crosstalk measurement indicated by the DETECTED mask plane.
177  debug.display['pixels'] : `bool`
178  Display a plot of the ratio calculated for each pixel used in this
179  exposure, split by amplifier pairs. The median value is listed
180  for reference.
181  """
182  outputRatios = defaultdict(lambda: defaultdict(dict))
183  outputFluxes = defaultdict(lambda: defaultdict(dict))
184 
185  threshold = self.config.threshold
186  badPixels = list(self.config.badMask)
187 
188  targetDetector = inputExp.getDetector()
189  targetChip = targetDetector.getName()
190 
191  # Always look at the target chip first, then go to any other supplied exposures.
192  sourceExtractExps = [inputExp]
193  sourceExtractExps.extend(sourceExps)
194 
195  self.log.info("Measuring full detector background for target: %s", targetChip)
196  targetIm = inputExp.getMaskedImage()
197  FootprintSet(targetIm, Threshold(threshold), "DETECTED")
198  detected = targetIm.getMask().getPlaneBitMask("DETECTED")
199  bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
200 
201  self.debugViewdebugView('extract', inputExp)
202 
203  for sourceExp in sourceExtractExps:
204  sourceDetector = sourceExp.getDetector()
205  sourceChip = sourceDetector.getName()
206  sourceIm = sourceExp.getMaskedImage()
207  bad = sourceIm.getMask().getPlaneBitMask(badPixels)
208  self.log.info("Measuring crosstalk from source: %s", sourceChip)
209 
210  if sourceExp != inputExp:
211  FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
212  detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
213 
214  # The dictionary of amp-to-amp ratios for this pair of source->target detectors.
215  ratioDict = defaultdict(lambda: defaultdict(list))
216  extractedCount = 0
217 
218  for sourceAmp in sourceDetector:
219  sourceAmpName = sourceAmp.getName()
220  sourceAmpBBox = sourceAmp.getBBox() if self.config.isTrimmed else sourceAmp.getRawDataBBox()
221  sourceAmpImage = sourceIm[sourceAmpBBox]
222  sourceMask = sourceAmpImage.mask.array
223  select = ((sourceMask & detected > 0)
224  & (sourceMask & bad == 0)
225  & np.isfinite(sourceAmpImage.image.array))
226  count = np.sum(select)
227  self.log.debug(" Source amplifier: %s", sourceAmpName)
228 
229  outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
230 
231  for targetAmp in targetDetector:
232  # iterate over targetExposure
233  targetAmpName = targetAmp.getName()
234  if sourceAmpName == targetAmpName and sourceChip == targetChip:
235  ratioDict[sourceAmpName][targetAmpName] = []
236  continue
237  self.log.debug(" Target amplifier: %s", targetAmpName)
238 
239  targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
240  targetAmp, sourceAmp,
241  isTrimmed=self.config.isTrimmed)
242  ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
243  ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
244  extractedCount += count
245 
246  self.debugPixelsdebugPixels('pixels',
247  sourceAmpImage.image.array[select],
248  targetAmpImage.array[select] - bg,
249  sourceAmpName, targetAmpName)
250 
251  self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
252  extractedCount, sourceChip, targetChip, bg)
253  outputRatios[targetChip][sourceChip] = ratioDict
254 
255  return pipeBase.Struct(
256  outputRatios=ddict2dict(outputRatios),
257  outputFluxes=ddict2dict(outputFluxes)
258  )
259 
260  def debugView(self, stepname, exposure):
261  """Utility function to examine the image being processed.
262 
263  Parameters
264  ----------
265  stepname : `str`
266  State of processing to view.
267  exposure : `lsst.afw.image.Exposure`
268  Exposure to view.
269  """
270  frame = getDebugFrame(self._display, stepname)
271  if frame:
272  display = getDisplay(frame)
273  display.scale('asinh', 'zscale')
274  display.mtv(exposure)
275 
276  prompt = "Press Enter to continue: "
277  while True:
278  ans = input(prompt).lower()
279  if ans in ("", "c",):
280  break
281 
282  def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
283  """Utility function to examine the CT ratio pixel values.
284 
285  Parameters
286  ----------
287  stepname : `str`
288  State of processing to view.
289  pixelsIn : `np.ndarray`
290  Pixel values from the potential crosstalk source.
291  pixelsOut : `np.ndarray`
292  Pixel values from the potential crosstalk target.
293  sourceName : `str`
294  Source amplifier name
295  targetName : `str`
296  Target amplifier name
297  """
298  frame = getDebugFrame(self._display, stepname)
299  if frame:
300  import matplotlib.pyplot as plt
301  figure = plt.figure(1)
302  figure.clear()
303 
304  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
305  axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+')
306  plt.xlabel("Source amplifier pixel value")
307  plt.ylabel("Measured pixel ratio")
308  plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: "
309  f"{(np.median(pixelsOut / pixelsIn))}")
310  figure.show()
311 
312  prompt = "Press Enter to continue: "
313  while True:
314  ans = input(prompt).lower()
315  if ans in ("", "c",):
316  break
317  plt.close()
318 
319 
320 class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections,
321  dimensions=("instrument", "detector")):
322  inputRatios = cT.Input(
323  name="crosstalkRatios",
324  doc="Ratios measured for an input exposure.",
325  storageClass="StructuredDataDict",
326  dimensions=("instrument", "exposure", "detector"),
327  multiple=True,
328  )
329  inputFluxes = cT.Input(
330  name="crosstalkFluxes",
331  doc="Fluxes of CT source pixels, for nonlinear fits.",
332  storageClass="StructuredDataDict",
333  dimensions=("instrument", "exposure", "detector"),
334  multiple=True,
335  )
336  camera = cT.PrerequisiteInput(
337  name="camera",
338  doc="Camera the input data comes from.",
339  storageClass="Camera",
340  dimensions=("instrument",),
341  isCalibration=True,
342  lookupFunction=lookupStaticCalibration,
343  )
344 
345  outputCrosstalk = cT.Output(
346  name="crosstalk",
347  doc="Output proposed crosstalk calibration.",
348  storageClass="CrosstalkCalib",
349  dimensions=("instrument", "detector"),
350  multiple=False,
351  isCalibration=True,
352  )
353 
354  def __init__(self, *, config=None):
355  super().__init__(config=config)
356 
357  if config.fluxOrder == 0:
358  self.inputs.discard("inputFluxes")
359 
360 
361 class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig,
362  pipelineConnections=CrosstalkSolveConnections):
363  """Configuration for the solving of crosstalk from pixel ratios.
364  """
365  rejIter = Field(
366  dtype=int,
367  default=3,
368  doc="Number of rejection iterations for final coefficient calculation.",
369  )
370  rejSigma = Field(
371  dtype=float,
372  default=2.0,
373  doc="Rejection threshold (sigma) for final coefficient calculation.",
374  )
375  fluxOrder = Field(
376  dtype=int,
377  default=0,
378  doc="Polynomial order in source flux to fit crosstalk.",
379  )
380  doFiltering = Field(
381  dtype=bool,
382  default=False,
383  doc="Filter generated crosstalk to remove marginal measurements.",
384  )
385 
386 
387 class CrosstalkSolveTask(pipeBase.PipelineTask,
388  pipeBase.CmdLineTask):
389  """Task to solve crosstalk from pixel ratios.
390  """
391  ConfigClass = CrosstalkSolveConfig
392  _DefaultName = 'cpCrosstalkSolve'
393 
394  def runQuantum(self, butlerQC, inputRefs, outputRefs):
395  """Ensure that the input and output dimensions are passed along.
396 
397  Parameters
398  ----------
399  butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
400  Butler to operate on.
401  inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
402  Input data refs to load.
403  ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
404  Output data refs to persist.
405  """
406  inputs = butlerQC.get(inputRefs)
407 
408  # Use the dimensions to set calib/provenance information.
409  inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios]
410  inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
411 
412  outputs = self.runrun(**inputs)
413  butlerQC.put(outputs, outputRefs)
414 
415  def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
416  """Combine ratios to produce crosstalk coefficients.
417 
418  Parameters
419  ----------
420  inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
421  A list of nested dictionaries of ratios indexed by target
422  and source chip, then by target and source amplifier.
423  inputFluxes : `list` [`dict` [`dict` [`list`]]]
424  A list of nested dictionaries of source pixel fluxes, indexed
425  by source chip and amplifier.
426  camera : `lsst.afw.cameraGeom.Camera`
427  Input camera.
428  inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
429  DataIds to use to construct provenance.
430  outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
431  DataIds to use to populate the output calibration.
432 
433  Returns
434  -------
435  results : `lsst.pipe.base.Struct`
436  The results struct containing:
437 
438  ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
439  Final crosstalk calibration.
440  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
441  Provenance data for the new calibration.
442 
443  Raises
444  ------
445  RuntimeError
446  Raised if the input data contains multiple target detectors.
447 
448  Notes
449  -----
450  The lsstDebug.Info() method can be rewritten for __name__ =
451  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
452 
453  debug.display['reduce'] : `bool`
454  Display a histogram of the combined ratio measurements for
455  a pair of source/target amplifiers from all input
456  exposures/detectors.
457 
458  """
459  if outputDims:
460  calibChip = outputDims['detector']
461  instrument = outputDims['instrument']
462  else:
463  # calibChip needs to be set manually in Gen2.
464  calibChip = None
465  instrument = None
466 
467  self.log.info("Combining measurements from %d ratios and %d fluxes",
468  len(inputRatios), len(inputFluxes) if inputFluxes else 0)
469 
470  if inputFluxes is None:
471  inputFluxes = [None for exp in inputRatios]
472 
473  combinedRatios = defaultdict(lambda: defaultdict(list))
474  combinedFluxes = defaultdict(lambda: defaultdict(list))
475  for ratioDict, fluxDict in zip(inputRatios, inputFluxes):
476  for targetChip in ratioDict:
477  if calibChip and targetChip != calibChip:
478  raise RuntimeError("Received multiple target chips!")
479 
480  sourceChip = targetChip
481  if sourceChip in ratioDict[targetChip]:
482  ratios = ratioDict[targetChip][sourceChip]
483 
484  for targetAmp in ratios:
485  for sourceAmp in ratios[targetAmp]:
486  combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
487  if fluxDict:
488  combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
489  # TODO: DM-21904
490  # Iterating over all other entries in ratioDict[targetChip] will yield
491  # inter-chip terms.
492 
493  for targetAmp in combinedRatios:
494  for sourceAmp in combinedRatios[targetAmp]:
495  self.log.info("Read %d pixels for %s -> %s",
496  len(combinedRatios[targetAmp][sourceAmp]),
497  targetAmp, sourceAmp)
498  if len(combinedRatios[targetAmp][sourceAmp]) > 1:
499  self.debugRatiosdebugRatios('reduce', combinedRatios, targetAmp, sourceAmp)
500 
501  if self.config.fluxOrder == 0:
502  self.log.info("Fitting crosstalk coefficients.")
503  calib = self.measureCrosstalkCoefficientsmeasureCrosstalkCoefficients(combinedRatios,
504  self.config.rejIter, self.config.rejSigma)
505  else:
506  raise NotImplementedError("Non-linear crosstalk terms are not yet supported.")
507 
508  self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid))
509 
510  if self.config.doFiltering:
511  # This step will apply the calculated validity values to
512  # censor poorly measured coefficients.
513  self.log.info("Filtering measured crosstalk to remove invalid solutions.")
514  calib = self.filterCrosstalkCalibfilterCrosstalkCalib(calib)
515 
516  # Populate the remainder of the calibration information.
517  calib.hasCrosstalk = True
518  calib.interChip = {}
519 
520  # calibChip is the detector dimension, which is the detector Id
521  calib._detectorId = calibChip
522  if camera:
523  calib._detectorName = camera[calibChip].getName()
524  calib._detectorSerial = camera[calibChip].getSerial()
525 
526  calib._instrument = instrument
527  calib.updateMetadata(setCalibId=True, setDate=True)
528 
529  # Make an IsrProvenance().
530  provenance = IsrProvenance(calibType="CROSSTALK")
531  provenance._detectorName = calibChip
532  if inputDims:
533  provenance.fromDataIds(inputDims)
534  provenance._instrument = instrument
535  provenance.updateMetadata()
536 
537  return pipeBase.Struct(
538  outputCrosstalk=calib,
539  outputProvenance=provenance,
540  )
541 
542  def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma):
543  """Measure crosstalk coefficients from the ratios.
544 
545  Given a list of ratios for each target/source amp combination,
546  we measure a sigma clipped mean and error.
547 
548  The coefficient errors returned are the standard deviation of
549  the final set of clipped input ratios.
550 
551  Parameters
552  ----------
553  ratios : `dict` of `dict` of `numpy.ndarray`
554  Catalog of arrays of ratios.
555  rejIter : `int`
556  Number of rejection iterations.
557  rejSigma : `float`
558  Rejection threshold (sigma).
559 
560  Returns
561  -------
562  calib : `lsst.ip.isr.CrosstalkCalib`
563  The output crosstalk calibration.
564 
565  Notes
566  -----
567  The lsstDebug.Info() method can be rewritten for __name__ =
568  `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
569 
570  debug.display['measure'] : `bool`
571  Display the CDF of the combined ratio measurements for
572  a pair of source/target amplifiers from the final set of
573  clipped input ratios.
574  """
575  calib = CrosstalkCalib(nAmp=len(ratios))
576 
577  # Calibration stores coefficients as a numpy ndarray.
578  ordering = list(ratios.keys())
579  for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
580  if ii == jj:
581  values = [0.0]
582  else:
583  values = np.array(ratios[ordering[ii]][ordering[jj]])
584  values = values[np.abs(values) < 1.0] # Discard unreasonable values
585 
586  calib.coeffNum[ii][jj] = len(values)
587 
588  if len(values) == 0:
589  self.log.warn("No values for matrix element %d,%d" % (ii, jj))
590  calib.coeffs[ii][jj] = np.nan
591  calib.coeffErr[ii][jj] = np.nan
592  calib.coeffValid[ii][jj] = False
593  else:
594  if ii != jj:
595  for rej in range(rejIter):
596  lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
597  sigma = 0.741*(hi - lo)
598  good = np.abs(values - med) < rejSigma*sigma
599  if good.sum() == len(good):
600  break
601  values = values[good]
602 
603  calib.coeffs[ii][jj] = np.mean(values)
604  if calib.coeffNum[ii][jj] == 1:
605  calib.coeffErr[ii][jj] = np.nan
606  else:
607  correctionFactor = sigmaClipCorrection(rejSigma)
608  calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
609  calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj])
610  > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
611 
612  if calib.coeffNum[ii][jj] > 1:
613  self.debugRatiosdebugRatios('measure', ratios, ordering[ii], ordering[jj],
614  calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
615 
616  return calib
617 
618  @staticmethod
619  def filterCrosstalkCalib(inCalib):
620  """Apply valid constraints to the measured values.
621 
622  Any measured coefficient that is determined to be invalid is
623  set to zero, and has the error set to nan. The validation is
624  determined by checking that the measured coefficient is larger
625  than the calculated standard error of the mean.
626 
627  Parameters
628  ----------
629  inCalib : `lsst.ip.isr.CrosstalkCalib`
630  Input calibration to filter.
631 
632  Returns
633  -------
634  outCalib : `lsst.ip.isr.CrosstalkCalib`
635  Filtered calibration.
636  """
637  outCalib = CrosstalkCalib()
638  outCalib.numAmps = inCalib.numAmps
639 
640  outCalib.coeffs = inCalib.coeffs
641  outCalib.coeffs[~inCalib.coeffValid] = 0.0
642 
643  outCalib.coeffErr = inCalib.coeffErr
644  outCalib.coeffErr[~inCalib.coeffValid] = np.nan
645 
646  outCalib.coeffNum = inCalib.coeffNum
647  outCalib.coeffValid = inCalib.coeffValid
648 
649  return outCalib
650 
651  def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
652  """Utility function to examine the final CT ratio set.
653 
654  Parameters
655  ----------
656  stepname : `str`
657  State of processing to view.
658  ratios : `dict` of `dict` of `np.ndarray`
659  Array of measured CT ratios, indexed by source/victim
660  amplifier.
661  i : `str`
662  Index of the source amplifier.
663  j : `str`
664  Index of the target amplifier.
665  coeff : `float`, optional
666  Coefficient calculated to plot along with the simple mean.
667  valid : `bool`, optional
668  Validity to be added to the plot title.
669  """
670  frame = getDebugFrame(self._display, stepname)
671  if frame:
672  if i == j or ratios is None or len(ratios) < 1:
673  pass
674 
675  ratioList = ratios[i][j]
676  if ratioList is None or len(ratioList) < 1:
677  pass
678 
679  mean = np.mean(ratioList)
680  std = np.std(ratioList)
681  import matplotlib.pyplot as plt
682  figure = plt.figure(1)
683  figure.clear()
684  plt.hist(x=ratioList, bins=len(ratioList),
685  cumulative=True, color='b', density=True, histtype='step')
686  plt.xlabel("Measured pixel ratio")
687  plt.ylabel(f"CDF: n={len(ratioList)}")
688  plt.xlim(np.percentile(ratioList, [1.0, 99]))
689  plt.axvline(x=mean, color="k")
690  plt.axvline(x=coeff, color='g')
691  plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r')
692  plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r')
693  plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
694  figure.show()
695 
696  prompt = "Press Enter to continue: "
697  while True:
698  ans = input(prompt).lower()
699  if ans in ("", "c",):
700  break
701  elif ans in ("pdb", "p",):
702  import pdb
703  pdb.set_trace()
704  plt.close()
705 
706 
708  extract = ConfigurableField(
709  target=CrosstalkExtractTask,
710  doc="Task to measure pixel ratios.",
711  )
712  solver = ConfigurableField(
713  target=CrosstalkSolveTask,
714  doc="Task to convert ratio lists to crosstalk coefficients.",
715  )
716 
717 
718 class MeasureCrosstalkTask(pipeBase.CmdLineTask):
719  """Measure intra-detector crosstalk.
720 
721  See also
722  --------
723  lsst.ip.isr.crosstalk.CrosstalkCalib
724  lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
725  lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
726 
727  Notes
728  -----
729  The crosstalk this method measures assumes that when a bright
730  pixel is found in one detector amplifier, all other detector
731  amplifiers may see a signal change in the same pixel location
732  (relative to the readout amplifier) as these other pixels are read
733  out at the same time.
734 
735  After processing each input exposure through a limited set of ISR
736  stages, bright unmasked pixels above the threshold are identified.
737  The potential CT signal is found by taking the ratio of the
738  appropriate background-subtracted pixel value on the other
739  amplifiers to the input value on the source amplifier. If the
740  source amplifier has a large number of bright pixels as well, the
741  background level may be elevated, leading to poor ratio
742  measurements.
743 
744  The set of ratios found between each pair of amplifiers across all
745  input exposures is then gathered to produce the final CT
746  coefficients. The sigma-clipped mean and sigma are returned from
747  these sets of ratios, with the coefficient to supply to the ISR
748  CrosstalkTask() being the multiplicative inverse of these values.
749 
750  This Task simply calls the pipetask versions of the measure
751  crosstalk code.
752  """
753  ConfigClass = MeasureCrosstalkConfig
754  _DefaultName = "measureCrosstalk"
755 
756  # Let's use this instead of messing with parseAndRun.
757  RunnerClass = DataRefListRunner
758 
759  def __init__(self, **kwargs):
760  super().__init__(**kwargs)
761  self.makeSubtask("extract")
762  self.makeSubtask("solver")
763 
764  def runDataRef(self, dataRefList):
765  """Run extract task on each of inputs in the dataRef list, then pass
766  that to the solver task.
767 
768  Parameters
769  ----------
770  dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
771  Data references for exposures for detectors to process.
772 
773  Returns
774  -------
775  results : `lsst.pipe.base.Struct`
776  The results struct containing:
777 
778  ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
779  Final crosstalk calibration.
780  ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
781  Provenance data for the new calibration.
782 
783  Raises
784  ------
785  RuntimeError
786  Raised if multiple target detectors are supplied.
787  """
788  dataRef = dataRefList[0]
789  camera = dataRef.get("camera")
790 
791  ratios = []
792  activeChip = None
793  for dataRef in dataRefList:
794  exposure = dataRef.get("postISRCCD")
795  if activeChip:
796  if exposure.getDetector().getName() != activeChip:
797  raise RuntimeError("Too many input detectors supplied!")
798  else:
799  activeChip = exposure.getDetector().getName()
800 
801  self.extract.debugView("extract", exposure)
802  result = self.extract.run(exposure)
803  ratios.append(result.outputRatios)
804 
805  for detIter, detector in enumerate(camera):
806  if detector.getName() == activeChip:
807  detectorId = detIter
808  outputDims = {'instrument': camera.getName(),
809  'detector': detectorId,
810  }
811 
812  finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims)
813  dataRef.put(finalResults.outputCrosstalk, "crosstalk")
814 
815  return finalResults
def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName)
def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None)
def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False)
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def sigmaClipCorrection(nSigClip)
Definition: utils.py:42
def ddict2dict(d)
Definition: utils.py:797