Coverage for python/lsst/cp/pipe/measureCrosstalk.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import itertools
22import numpy as np
23from scipy.stats import norm
25from collections import defaultdict
27import lsst.pipe.base as pipeBase
28import lsst.pipe.base.connectionTypes as cT
30from lsstDebug import getDebugFrame
31from lsst.afw.detection import FootprintSet, Threshold
32from lsst.afw.display import getDisplay
33from lsst.pex.config import Config, Field, ListField, ConfigurableField
34from lsst.ip.isr import CrosstalkCalib, IsrProvenance
35from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
37__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask",
38 "CrosstalkSolveTask", "CrosstalkSolveConfig",
39 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
42class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "exposure", "detector")):
44 inputExp = cT.Input(
45 name="crosstalkInputs",
46 doc="Input post-ISR processed exposure to measure crosstalk from.",
47 storageClass="Exposure",
48 dimensions=("instrument", "exposure", "detector"),
49 multiple=False,
50 )
51 # TODO: Depends on DM-21904.
52 sourceExp = cT.Input(
53 name="crosstalkSource",
54 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
55 storageClass="Exposure",
56 dimensions=("instrument", "exposure", "detector"),
57 multiple=True,
58 deferLoad=True,
59 # lookupFunction=None,
60 )
62 outputRatios = cT.Output(
63 name="crosstalkRatios",
64 doc="Extracted crosstalk pixel ratios.",
65 storageClass="StructuredDataDict",
66 dimensions=("instrument", "exposure", "detector"),
67 )
68 outputFluxes = cT.Output(
69 name="crosstalkFluxes",
70 doc="Source pixel fluxes used in ratios.",
71 storageClass="StructuredDataDict",
72 dimensions=("instrument", "exposure", "detector"),
73 )
75 def __init__(self, *, config=None):
76 super().__init__(config=config)
77 # Discard sourceExp until DM-21904 allows full interchip
78 # measurements.
79 self.inputs.discard("sourceExp")
82class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig,
83 pipelineConnections=CrosstalkExtractConnections):
84 """Configuration for the measurement of pixel ratios.
85 """
86 doMeasureInterchip = Field(
87 dtype=bool,
88 default=False,
89 doc="Measure inter-chip crosstalk as well?",
90 )
91 threshold = Field(
92 dtype=float,
93 default=30000,
94 doc="Minimum level of source pixels for which to measure crosstalk."
95 )
96 ignoreSaturatedPixels = Field(
97 dtype=bool,
98 default=True,
99 doc="Should saturated pixels be ignored?"
100 )
101 badMask = ListField(
102 dtype=str,
103 default=["BAD", "INTRP"],
104 doc="Mask planes to ignore when identifying source pixels."
105 )
106 isTrimmed = Field(
107 dtype=bool,
108 default=True,
109 doc="Is the input exposure trimmed?"
110 )
112 def validate(self):
113 super().validate()
115 # Ensure the handling of the SAT mask plane is consistent
116 # with the ignoreSaturatedPixels value.
117 if self.ignoreSaturatedPixels:
118 if 'SAT' not in self.badMask:
119 self.badMask.append('SAT')
120 else:
121 if 'SAT' in self.badMask:
122 self.badMask = [mask for mask in self.badMask if mask != 'SAT']
125class CrosstalkExtractTask(pipeBase.PipelineTask,
126 pipeBase.CmdLineTask):
127 """Task to measure pixel ratios to find crosstalk.
128 """
129 ConfigClass = CrosstalkExtractConfig
130 _DefaultName = 'cpCrosstalkExtract'
132 def run(self, inputExp, sourceExps=[]):
133 """Measure pixel ratios between amplifiers in inputExp.
135 Extract crosstalk ratios between different amplifiers.
137 For pixels above ``config.threshold``, we calculate the ratio
138 between each background-subtracted target amp and the source
139 amp. We return a list of ratios for each pixel for each
140 target/source combination, as nested dictionary containing the
141 ratio.
143 Parameters
144 ----------
145 inputExp : `lsst.afw.image.Exposure`
146 Input exposure to measure pixel ratios on.
147 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
148 List of chips to use as sources to measure inter-chip
149 crosstalk.
151 Returns
152 -------
153 results : `lsst.pipe.base.Struct`
154 The results struct containing:
156 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
157 A catalog of ratio lists. The dictionaries are
158 indexed such that:
159 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
160 contains the ratio list for that combination.
161 ``outputFluxes`` : `dict` [`dict` [`list`]]
162 A catalog of flux lists. The dictionaries are
163 indexed such that:
164 outputFluxes[sourceChip][sourceAmp]
165 contains the flux list used in the outputRatios.
167 Notes
168 -----
169 The lsstDebug.Info() method can be rewritten for __name__ =
170 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
172 debug.display['extract'] : `bool`
173 Display the exposure under consideration, with the pixels used
174 for crosstalk measurement indicated by the DETECTED mask plane.
175 debug.display['pixels'] : `bool`
176 Display a plot of the ratio calculated for each pixel used in this
177 exposure, split by amplifier pairs. The median value is listed
178 for reference.
179 """
180 outputRatios = defaultdict(lambda: defaultdict(dict))
181 outputFluxes = defaultdict(lambda: defaultdict(dict))
183 threshold = self.config.threshold
184 badPixels = list(self.config.badMask)
186 targetDetector = inputExp.getDetector()
187 targetChip = targetDetector.getName()
189 # Always look at the target chip first, then go to any other supplied exposures.
190 sourceExtractExps = [inputExp]
191 sourceExtractExps.extend(sourceExps)
193 self.log.info("Measuring full detector background for target: %s", targetChip)
194 targetIm = inputExp.getMaskedImage()
195 FootprintSet(targetIm, Threshold(threshold), "DETECTED")
196 detected = targetIm.getMask().getPlaneBitMask("DETECTED")
197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
199 self.debugView('extract', inputExp)
201 for sourceExp in sourceExtractExps:
202 sourceDetector = sourceExp.getDetector()
203 sourceChip = sourceDetector.getName()
204 sourceIm = sourceExp.getMaskedImage()
205 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
206 self.log.info("Measuring crosstalk from source: %s", sourceChip)
208 if sourceExp != inputExp:
209 FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
210 detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
212 # The dictionary of amp-to-amp ratios for this pair of source->target detectors.
213 ratioDict = defaultdict(lambda: defaultdict(list))
214 extractedCount = 0
216 for sourceAmp in sourceDetector:
217 sourceAmpName = sourceAmp.getName()
218 sourceAmpImage = sourceIm[sourceAmp.getBBox()]
219 sourceMask = sourceAmpImage.mask.array
220 select = ((sourceMask & detected > 0) &
221 (sourceMask & bad == 0) &
222 np.isfinite(sourceAmpImage.image.array))
223 count = np.sum(select)
224 self.log.debug(" Source amplifier: %s", sourceAmpName)
226 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select]
228 for targetAmp in targetDetector:
229 # iterate over targetExposure
230 targetAmpName = targetAmp.getName()
231 if sourceAmpName == targetAmpName and sourceChip == targetChip:
232 ratioDict[sourceAmpName][targetAmpName] = []
233 continue
234 self.log.debug(" Target amplifier: %s", targetAmpName)
236 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
237 targetAmp, sourceAmp,
238 isTrimmed=self.config.isTrimmed)
239 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
240 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
241 extractedCount += count
243 self.debugPixels('pixels',
244 sourceAmpImage.image.array[select],
245 targetAmpImage.array[select] - bg,
246 sourceAmpName, targetAmpName)
248 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
249 extractedCount, sourceChip, targetChip, bg)
250 outputRatios[targetChip][sourceChip] = ratioDict
252 return pipeBase.Struct(
253 outputRatios=outputRatios,
254 outputFluxes=outputFluxes
255 )
257 def debugView(self, stepname, exposure):
258 """Utility function to examine the image being processed.
260 Parameters
261 ----------
262 stepname : `str`
263 State of processing to view.
264 exposure : `lsst.afw.image.Exposure`
265 Exposure to view.
266 """
267 frame = getDebugFrame(self._display, stepname)
268 if frame:
269 display = getDisplay(frame)
270 display.scale('asinh', 'zscale')
271 display.mtv(exposure)
273 prompt = "Press Enter to continue: "
274 while True:
275 ans = input(prompt).lower()
276 if ans in ("", "c",):
277 break
279 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
280 """Utility function to examine the CT ratio pixel values.
282 Parameters
283 ----------
284 stepname : `str`
285 State of processing to view.
286 pixelsIn : `np.ndarray`
287 Pixel values from the potential crosstalk source.
288 pixelsOut : `np.ndarray`
289 Pixel values from the potential crosstalk target.
290 sourceName : `str`
291 Source amplifier name
292 targetName : `str`
293 Target amplifier name
294 """
295 frame = getDebugFrame(self._display, stepname)
296 if frame:
297 import matplotlib.pyplot as plt
298 figure = plt.figure(1)
299 figure.clear()
301 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
302 axes.plt(pixelsIn, pixelsOut / pixelsIn, 'k+')
303 plt.xlabel("Source amplifier pixel value")
304 plt.ylabel("Measured pixel ratio")
305 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: "
306 f"{(np.median(pixelsOut / pixelsIn))}")
307 figure.show()
309 prompt = "Press Enter to continue: "
310 while True:
311 ans = input(prompt).lower()
312 if ans in ("", "c",):
313 break
314 plt.close()
317class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections,
318 dimensions=("instrument", "detector")):
319 inputRatios = cT.Input(
320 name="crosstalkRatios",
321 doc="Ratios measured for an input exposure.",
322 storageClass="StructuredDataDict",
323 dimensions=("instrument", "exposure", "detector"),
324 multiple=True,
325 )
326 inputFluxes = cT.Input(
327 name="crosstalkFluxes",
328 doc="Fluxes of CT source pixels, for nonlinear fits.",
329 storageClass="StructuredDataDict",
330 dimensions=("instrument", "exposure", "detector"),
331 multiple=True,
332 )
333 camera = cT.PrerequisiteInput(
334 name="camera",
335 doc="Camera the input data comes from.",
336 storageClass="Camera",
337 dimensions=("instrument", "calibration_label"),
338 )
340 outputCrosstalk = cT.Output(
341 name="crosstalkProposal",
342 doc="Output proposed crosstalk calibration.",
343 storageClass="CrosstalkCalib",
344 dimensions=("instrument", "detector"),
345 multiple=False,
346 )
348 def __init__(self, *, config=None):
349 super().__init__(config=config)
351 if config.fluxOrder == 0:
352 self.Inputs.discard("inputFluxes")
355class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig,
356 pipelineConnections=CrosstalkSolveConnections):
357 """Configuration for the solving of crosstalk from pixel ratios.
358 """
359 rejIter = Field(
360 dtype=int,
361 default=3,
362 doc="Number of rejection iterations for final coefficient calculation.",
363 )
364 rejSigma = Field(
365 dtype=float,
366 default=2.0,
367 doc="Rejection threshold (sigma) for final coefficient calculation.",
368 )
369 fluxOrder = Field(
370 dtype=int,
371 default=0,
372 doc="Polynomial order in source flux to fit crosstalk.",
373 )
374 doFiltering = Field(
375 dtype=bool,
376 default=False,
377 doc="Filter generated crosstalk to remove marginal measurements.",
378 )
381class CrosstalkSolveTask(pipeBase.PipelineTask,
382 pipeBase.CmdLineTask):
383 """Task to solve crosstalk from pixel ratios.
384 """
385 ConfigClass = CrosstalkSolveConfig
386 _DefaultName = 'cpCrosstalkSolve'
388 def runQuantum(self, butlerQC, inputRefs, outputRefs):
389 """Ensure that the input and output dimensions are passed along.
391 Parameters
392 ----------
393 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
394 Butler to operate on.
395 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
396 Input data refs to load.
397 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
398 Output data refs to persist.
399 """
400 inputs = butlerQC.get(inputRefs)
402 # Use the dimensions to set calib/provenance information.
403 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios]
404 inputs['outputDims'] = [exp.dataId.byName() for exp in outputRefs.outputCrosstalk]
406 outputs = self.run(**inputs)
407 butlerQC.put(outputs, outputRefs)
409 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
410 """Combine ratios to produce crosstalk coefficients.
412 Parameters
413 ----------
414 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
415 A list of nested dictionaries of ratios indexed by target
416 and source chip, then by target and source amplifier.
417 inputFluxes : `list` [`dict` [`dict` [`list`]]]
418 A list of nested dictionaries of source pixel fluxes, indexed
419 by source chip and amplifier.
420 camera : `lsst.afw.cameraGeom.Camera`
421 Input camera.
422 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
423 DataIds to use to construct provenance.
424 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
425 DataIds to use to populate the output calibration.
427 Returns
428 -------
429 results : `lsst.pipe.base.Struct`
430 The results struct containing:
432 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
433 Final crosstalk calibration.
434 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
435 Provenance data for the new calibration.
437 Raises
438 ------
439 RuntimeError
440 Raised if the input data contains multiple target detectors.
442 Notes
443 -----
444 The lsstDebug.Info() method can be rewritten for __name__ =
445 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
447 debug.display['reduce'] : `bool`
448 Display a histogram of the combined ratio measurements for
449 a pair of source/target amplifiers from all input
450 exposures/detectors.
452 """
453 if outputDims:
454 calibChip = outputDims['detector']
455 else:
456 # calibChip needs to be set manually in Gen2.
457 calibChip = None
459 self.log.info("Combining measurements from %d ratios and %d fluxes",
460 len(inputRatios), len(inputFluxes) if inputFluxes else 0)
462 if inputFluxes is None:
463 inputFluxes = [None for exp in inputRatios]
465 combinedRatios = defaultdict(lambda: defaultdict(list))
466 combinedFluxes = defaultdict(lambda: defaultdict(list))
467 for ratioDict, fluxDict in zip(inputRatios, inputFluxes):
468 for targetChip in ratioDict:
469 if calibChip and targetChip != calibChip:
470 raise RuntimeError("Received multiple target chips!")
472 sourceChip = targetChip
473 if sourceChip in ratioDict[targetChip]:
474 ratios = ratioDict[targetChip][sourceChip]
476 for targetAmp in ratios:
477 for sourceAmp in ratios[targetAmp]:
478 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
479 if fluxDict:
480 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
481 # TODO: DM-21904
482 # Iterating over all other entries in ratioDict[targetChip] will yield
483 # inter-chip terms.
485 for targetAmp in combinedRatios:
486 for sourceAmp in combinedRatios[targetAmp]:
487 self.log.info("Read %d pixels for %s -> %s",
488 len(combinedRatios[targetAmp][sourceAmp]),
489 targetAmp, sourceAmp)
490 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
491 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp)
493 if self.config.fluxOrder == 0:
494 self.log.info("Fitting crosstalk coefficients.")
495 calib = self.measureCrosstalkCoefficients(combinedRatios,
496 self.config.rejIter, self.config.rejSigma)
497 else:
498 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.")
500 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid))
502 if self.config.doFiltering:
503 # This step will apply the calculated validity values to
504 # censor poorly measured coefficients.
505 self.log.info("Filtering measured crosstalk to remove invalid solutions.")
506 calib = self.filterCrosstalkCalib(calib)
508 # Populate the remainder of the calibration information.
509 calib.hasCrosstalk = True
510 calib.interChip = {}
511 calib._detectorName = calibChip
512 if camera:
513 for chip in camera:
514 if chip.getName() == calibChip:
515 calib._detectorSerial = chip.getSerial()
516 calib.updateMetadata()
518 # Make an IsrProvenance().
519 provenance = IsrProvenance(calibType="CROSSTALK")
520 provenance._detectorName = calibChip
521 if inputDims:
522 provenance.fromDataIds(inputDims)
523 provenance._instrument = outputDims['instrument']
524 provenance.updateMetadata()
526 return pipeBase.Struct(
527 outputCrosstalk=calib,
528 outputProvenance=provenance,
529 )
531 def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma):
532 """Measure crosstalk coefficients from the ratios.
534 Given a list of ratios for each target/source amp combination,
535 we measure a sigma clipped mean and error.
537 The coefficient errors returned are the standard deviation of
538 the final set of clipped input ratios.
540 Parameters
541 ----------
542 ratios : `dict` of `dict` of `numpy.ndarray`
543 Catalog of arrays of ratios.
544 rejIter : `int`
545 Number of rejection iterations.
546 rejSigma : `float`
547 Rejection threshold (sigma).
549 Returns
550 -------
551 calib : `lsst.ip.isr.CrosstalkCalib`
552 The output crosstalk calibration.
554 Notes
555 -----
556 The lsstDebug.Info() method can be rewritten for __name__ =
557 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
559 debug.display['measure'] : `bool`
560 Display the CDF of the combined ratio measurements for
561 a pair of source/target amplifiers from the final set of
562 clipped input ratios.
563 """
564 calib = CrosstalkCalib(nAmp=len(ratios))
566 # Calibration stores coefficients as a numpy ndarray.
567 ordering = list(ratios.keys())
568 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
569 if ii == jj:
570 values = [0.0]
571 else:
572 values = np.array(ratios[ordering[ii]][ordering[jj]])
573 values = values[np.abs(values) < 1.0] # Discard unreasonable values
575 calib.coeffNum[ii][jj] = len(values)
577 if len(values) == 0:
578 self.log.warn("No values for matrix element %d,%d" % (ii, jj))
579 calib.coeffs[ii][jj] = np.nan
580 calib.coeffErr[ii][jj] = np.nan
581 calib.coeffValid[ii][jj] = False
582 else:
583 if ii != jj:
584 for rej in range(rejIter):
585 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
586 sigma = 0.741*(hi - lo)
587 good = np.abs(values - med) < rejSigma*sigma
588 if good.sum() == len(good):
589 break
590 values = values[good]
592 calib.coeffs[ii][jj] = np.mean(values)
593 if calib.coeffNum[ii][jj] == 1:
594 calib.coeffErr[ii][jj] = np.nan
595 else:
596 correctionFactor = self.sigmaClipCorrection(rejSigma)
597 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
598 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) >
599 calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
601 if calib.coeffNum[ii][jj] > 1:
602 self.debugRatios('measure', ratios, ordering[ii], ordering[jj],
603 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
605 return calib
607 @staticmethod
608 def sigmaClipCorrection(nSigClip):
609 """Correct measured sigma to account for clipping.
611 If we clip our input data and then measure sigma, then the
612 measured sigma is smaller than the true value because real
613 points beyond the clip threshold have been removed. This is a
614 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the
615 default parameters for measure crosstalk use nSigClip=2.0.
616 This causes the measured sigma to be about 15% smaller than
617 real. This formula corrects the issue, for the symmetric case
618 (upper clip threshold equal to lower clip threshold).
620 Parameters
621 ----------
622 nSigClip : `float`
623 Number of sigma the measurement was clipped by.
625 Returns
626 -------
627 scaleFactor : `float`
628 Scale factor to increase the measured sigma by.
630 """
631 varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip))
632 return 1.0 / np.sqrt(varFactor)
634 @staticmethod
635 def filterCrosstalkCalib(inCalib):
636 """Apply valid constraints to the measured values.
638 Any measured coefficient that is determined to be invalid is
639 set to zero, and has the error set to nan. The validation is
640 determined by checking that the measured coefficient is larger
641 than the calculated standard error of the mean.
643 Parameters
644 ----------
645 inCalib : `lsst.ip.isr.CrosstalkCalib`
646 Input calibration to filter.
648 Returns
649 -------
650 outCalib : `lsst.ip.isr.CrosstalkCalib`
651 Filtered calibration.
652 """
653 outCalib = CrosstalkCalib()
654 outCalib.numAmps = inCalib.numAmps
656 outCalib.coeffs = inCalib.coeffs
657 outCalib.coeffs[~inCalib.coeffValid] = 0.0
659 outCalib.coeffErr = inCalib.coeffErr
660 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
662 outCalib.coeffNum = inCalib.coeffNum
663 outCalib.coeffValid = inCalib.coeffValid
665 return outCalib
667 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
668 """Utility function to examine the final CT ratio set.
670 Parameters
671 ----------
672 stepname : `str`
673 State of processing to view.
674 ratios : `dict` of `dict` of `np.ndarray`
675 Array of measured CT ratios, indexed by source/victim
676 amplifier.
677 i : `str`
678 Index of the source amplifier.
679 j : `str`
680 Index of the target amplifier.
681 coeff : `float`, optional
682 Coefficient calculated to plot along with the simple mean.
683 valid : `bool`, optional
684 Validity to be added to the plot title.
685 """
686 frame = getDebugFrame(self._display, stepname)
687 if frame:
688 if i == j or ratios is None or len(ratios) < 1:
689 pass
691 ratioList = ratios[i][j]
692 if ratioList is None or len(ratioList) < 1:
693 pass
695 mean = np.mean(ratioList)
696 std = np.std(ratioList)
697 import matplotlib.pyplot as plt
698 figure = plt.figure(1)
699 figure.clear()
700 plt.hist(x=ratioList, bins=len(ratioList),
701 cumulative=True, color='b', density=True, histtype='step')
702 plt.xlabel("Measured pixel ratio")
703 plt.ylabel(f"CDF: n={len(ratioList)}")
704 plt.xlim(np.percentile(ratioList, [1.0, 99]))
705 plt.axvline(x=mean, color="k")
706 plt.axvline(x=coeff, color='g')
707 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r')
708 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r')
709 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
710 figure.show()
712 prompt = "Press Enter to continue: "
713 while True:
714 ans = input(prompt).lower()
715 if ans in ("", "c",):
716 break
717 elif ans in ("pdb", "p",):
718 import pdb
719 pdb.set_trace()
720 plt.close()
723class MeasureCrosstalkConfig(Config):
724 extract = ConfigurableField(
725 target=CrosstalkExtractTask,
726 doc="Task to measure pixel ratios.",
727 )
728 solver = ConfigurableField(
729 target=CrosstalkSolveTask,
730 doc="Task to convert ratio lists to crosstalk coefficients.",
731 )
734class MeasureCrosstalkTask(pipeBase.CmdLineTask):
735 """Measure intra-detector crosstalk.
737 See also
738 --------
739 lsst.ip.isr.crosstalk.CrosstalkCalib
740 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
741 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
743 Notes
744 -----
745 The crosstalk this method measures assumes that when a bright
746 pixel is found in one detector amplifier, all other detector
747 amplifiers may see a signal change in the same pixel location
748 (relative to the readout amplifier) as these other pixels are read
749 out at the same time.
751 After processing each input exposure through a limited set of ISR
752 stages, bright unmasked pixels above the threshold are identified.
753 The potential CT signal is found by taking the ratio of the
754 appropriate background-subtracted pixel value on the other
755 amplifiers to the input value on the source amplifier. If the
756 source amplifier has a large number of bright pixels as well, the
757 background level may be elevated, leading to poor ratio
758 measurements.
760 The set of ratios found between each pair of amplifiers across all
761 input exposures is then gathered to produce the final CT
762 coefficients. The sigma-clipped mean and sigma are returned from
763 these sets of ratios, with the coefficient to supply to the ISR
764 CrosstalkTask() being the multiplicative inverse of these values.
766 This Task simply calls the pipetask versions of the measure
767 crosstalk code.
768 """
769 ConfigClass = MeasureCrosstalkConfig
770 _DefaultName = "measureCrosstalk"
772 # Let's use this instead of messing with parseAndRun.
773 RunnerClass = DataRefListRunner
775 def __init__(self, **kwargs):
776 super().__init__(**kwargs)
777 self.makeSubtask("extract")
778 self.makeSubtask("solver")
780 def runDataRef(self, dataRefList):
781 """Run extract task on each of inputs in the dataRef list, then pass
782 that to the solver task.
784 Parameters
785 ----------
786 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
787 Data references for exposures for detectors to process.
789 Returns
790 -------
791 results : `lsst.pipe.base.Struct`
792 The results struct containing:
794 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
795 Final crosstalk calibration.
796 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
797 Provenance data for the new calibration.
799 Raises
800 ------
801 RuntimeError
802 Raised if multiple target detectors are supplied.
803 """
804 dataRef = dataRefList[0]
805 camera = dataRef.get("camera")
807 ratios = []
808 activeChip = None
809 for dataRef in dataRefList:
810 exposure = dataRef.get("postISRCCD")
811 if activeChip:
812 if exposure.getDetector().getName() != activeChip:
813 raise RuntimeError("Too many input detectors supplied!")
814 else:
815 activeChip = exposure.getDetector().getName()
817 self.extract.debugView("extract", exposure)
818 result = self.extract.run(exposure)
819 ratios.append(result.outputRatios)
821 finalResults = self.solver.run(ratios, camera=camera)
822 dataRef.put(finalResults.outputCrosstalk, "crosstalk")
824 return finalResults