Coverage for python/lsst/cp/pipe/measureCrosstalk.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import itertools
22import numpy as np
23from scipy.stats import norm
25from collections import defaultdict
27import lsst.pipe.base as pipeBase
28import lsst.pipe.base.connectionTypes as cT
30from lsstDebug import getDebugFrame
31from lsst.afw.detection import FootprintSet, Threshold
32from lsst.afw.display import getDisplay
33from lsst.pex.config import Config, Field, ListField, ConfigurableField
34from lsst.ip.isr import CrosstalkCalib, IsrProvenance
35from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
37__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask",
38 "CrosstalkSolveTask", "CrosstalkSolveConfig",
39 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
42class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "exposure", "detector")):
44 inputExp = cT.Input(
45 name="crosstalkInputs",
46 doc="Input post-ISR processed exposure to measure crosstalk from.",
47 storageClass="Exposure",
48 dimensions=("instrument", "exposure", "detector"),
49 multiple=False,
50 )
51 # TODO: Depends on DM-21904.
52 sourceExp = cT.Input(
53 name="crosstalkSource",
54 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
55 storageClass="Exposure",
56 dimensions=("instrument", "exposure", "detector"),
57 multiple=True,
58 deferLoad=True,
59 # lookupFunction=None,
60 )
62 outputRatios = cT.Output(
63 name="crosstalkRatios",
64 doc="Extracted crosstalk pixel ratios.",
65 storageClass="StructuredDataDict",
66 dimensions=("instrument", "exposure", "detector"),
67 )
68 outputFluxes = cT.Output(
69 name="crosstalkFluxes",
70 doc="Source pixel fluxes used in ratios.",
71 storageClass="StructuredDataDict",
72 dimensions=("instrument", "exposure", "detector"),
73 )
75 def __init__(self, *, config=None):
76 super().__init__(config=config)
77 # Discard sourceExp until DM-21904 allows full interchip
78 # measurements.
79 self.inputs.discard("sourceExp")
82class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig,
83 pipelineConnections=CrosstalkExtractConnections):
84 """Configuration for the measurement of pixel ratios.
85 """
86 doMeasureInterchip = Field(
87 dtype=bool,
88 default=False,
89 doc="Measure inter-chip crosstalk as well?",
90 )
91 threshold = Field(
92 dtype=float,
93 default=30000,
94 doc="Minimum level of source pixels for which to measure crosstalk."
95 )
96 ignoreSaturatedPixels = Field(
97 dtype=bool,
98 default=True,
99 doc="Should saturated pixels be ignored?"
100 )
101 badMask = ListField(
102 dtype=str,
103 default=["BAD", "INTRP"],
104 doc="Mask planes to ignore when identifying source pixels."
105 )
106 isTrimmed = Field(
107 dtype=bool,
108 default=True,
109 doc="Is the input exposure trimmed?"
110 )
112 def validate(self):
113 super().validate()
115 # Ensure the handling of the SAT mask plane is consistent
116 # with the ignoreSaturatedPixels value.
117 if self.ignoreSaturatedPixels:
118 if 'SAT' not in self.badMask:
119 self.badMask.append('SAT')
120 else:
121 if 'SAT' in self.badMask:
122 self.badMask = [mask for mask in self.badMask if mask != 'SAT']
125class CrosstalkExtractTask(pipeBase.PipelineTask,
126 pipeBase.CmdLineTask):
127 """Task to measure pixel ratios to find crosstalk.
128 """
129 ConfigClass = CrosstalkExtractConfig
130 _DefaultName = 'cpCrosstalkExtract'
132 def run(self, inputExp, sourceExps=[]):
133 """Measure pixel ratios between amplifiers in inputExp.
135 Extract crosstalk ratios between different amplifiers.
137 For pixels above ``config.threshold``, we calculate the ratio
138 between each background-subtracted target amp and the source
139 amp. We return a list of ratios for each pixel for each
140 target/source combination, as nested dictionary containing the
141 ratio.
143 Parameters
144 ----------
145 inputExp : `lsst.afw.image.Exposure`
146 Input exposure to measure pixel ratios on.
147 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
148 List of chips to use as sources to measure inter-chip
149 crosstalk.
151 Returns
152 -------
153 results : `lsst.pipe.base.Struct`
154 The results struct containing:
156 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
157 A catalog of ratio lists. The dictionaries are
158 indexed such that:
159 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
160 contains the ratio list for that combination.
161 ``outputFluxes`` : `dict` [`dict` [`list`]]
162 A catalog of flux lists. The dictionaries are
163 indexed such that:
164 outputFluxes[sourceChip][sourceAmp]
165 contains the flux list used in the outputRatios.
167 Notes
168 -----
169 The lsstDebug.Info() method can be rewritten for __name__ =
170 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
172 debug.display['extract'] : `bool`
173 Display the exposure under consideration, with the pixels used
174 for crosstalk measurement indicated by the DETECTED mask plane.
175 debug.display['pixels'] : `bool`
176 Display a plot of the ratio calculated for each pixel used in this
177 exposure, split by amplifier pairs. The median value is listed
178 for reference.
179 """
180 outputRatios = defaultdict(lambda: defaultdict(dict))
181 outputFluxes = defaultdict(lambda: defaultdict(dict))
183 threshold = self.config.threshold
184 badPixels = list(self.config.badMask)
186 targetDetector = inputExp.getDetector()
187 targetChip = targetDetector.getName()
189 # Always look at the target chip first, then go to any other supplied exposures.
190 sourceExtractExps = [inputExp]
191 sourceExtractExps.extend(sourceExps)
193 self.log.info("Measuring full detector background for target: %s", targetChip)
194 targetIm = inputExp.getMaskedImage()
195 FootprintSet(targetIm, Threshold(threshold), "DETECTED")
196 detected = targetIm.getMask().getPlaneBitMask("DETECTED")
197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
199 self.debugView('extract', inputExp)
201 for sourceExp in sourceExtractExps:
202 sourceDetector = sourceExp.getDetector()
203 sourceChip = sourceDetector.getName()
204 sourceIm = sourceExp.getMaskedImage()
205 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
206 self.log.info("Measuring crosstalk from source: %s", sourceChip)
208 if sourceExp != inputExp:
209 FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
210 detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
212 # The dictionary of amp-to-amp ratios for this pair of source->target detectors.
213 ratioDict = defaultdict(lambda: defaultdict(list))
214 extractedCount = 0
216 for sourceAmp in sourceDetector:
217 sourceAmpName = sourceAmp.getName()
218 sourceAmpImage = sourceIm[sourceAmp.getBBox()]
219 sourceMask = sourceAmpImage.mask.array
220 select = ((sourceMask & detected > 0) &
221 (sourceMask & bad == 0) &
222 np.isfinite(sourceAmpImage.image.array))
223 count = np.sum(select)
224 self.log.debug(" Source amplifier: %s", sourceAmpName)
226 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select]
228 for targetAmp in targetDetector:
229 # iterate over targetExposure
230 targetAmpName = targetAmp.getName()
231 if sourceAmpName == targetAmpName and sourceChip == targetChip:
232 ratioDict[sourceAmpName][targetAmpName] = []
233 continue
234 self.log.debug(" Target amplifier: %s", targetAmpName)
236 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
237 targetAmp, sourceAmp,
238 isTrimmed=self.config.isTrimmed)
239 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
240 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
241 extractedCount += count
243 self.debugPixels('pixels',
244 sourceAmpImage.image.array[select],
245 targetAmpImage.array[select] - bg,
246 sourceAmpName, targetAmpName)
248 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
249 extractedCount, sourceChip, targetChip, bg)
250 outputRatios[targetChip][sourceChip] = ratioDict
252 return pipeBase.Struct(
253 outputRatios=outputRatios,
254 outputFluxes=outputFluxes
255 )
257 def debugView(self, stepname, exposure):
258 """Utility function to examine the image being processed.
260 Parameters
261 ----------
262 stepname : `str`
263 State of processing to view.
264 exposure : `lsst.afw.image.Exposure`
265 Exposure to view.
266 """
267 frame = getDebugFrame(self._display, stepname)
268 if frame:
269 display = getDisplay(frame)
270 display.scale('asinh', 'zscale')
271 display.mtv(exposure)
273 prompt = "Press Enter to continue: "
274 while True:
275 ans = input(prompt).lower()
276 if ans in ("", "c",):
277 break
279 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
280 """Utility function to examine the CT ratio pixel values.
282 Parameters
283 ----------
284 stepname : `str`
285 State of processing to view.
286 pixelsIn : `np.ndarray`
287 Pixel values from the potential crosstalk source.
288 pixelsOut : `np.ndarray`
289 Pixel values from the potential crosstalk target.
290 sourceName : `str`
291 Source amplifier name
292 targetName : `str`
293 Target amplifier name
294 """
295 frame = getDebugFrame(self._display, stepname)
296 if frame:
297 import matplotlib.pyplot as plt
298 figure = plt.figure(1)
299 figure.clear()
301 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
302 axes.plt(pixelsIn, pixelsOut / pixelsIn, 'k+')
303 plt.xlabel("Source amplifier pixel value")
304 plt.ylabel("Measured pixel ratio")
305 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: "
306 f"{(np.median(pixelsOut / pixelsIn))}")
307 figure.show()
309 prompt = "Press Enter to continue: "
310 while True:
311 ans = input(prompt).lower()
312 if ans in ("", "c",):
313 break
314 plt.close()
317class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections,
318 dimensions=("instrument", "detector")):
319 inputRatios = cT.Input(
320 name="crosstalkRatios",
321 doc="Ratios measured for an input exposure.",
322 storageClass="StructuredDataDict",
323 dimensions=("instrument", "exposure", "detector"),
324 multiple=True,
325 )
326 inputFluxes = cT.Input(
327 name="crosstalkFluxes",
328 doc="Fluxes of CT source pixels, for nonlinear fits.",
329 storageClass="StructuredDataDict",
330 dimensions=("instrument", "exposure", "detector"),
331 multiple=True,
332 )
333 camera = cT.PrerequisiteInput(
334 name="camera",
335 doc="Camera the input data comes from.",
336 storageClass="Camera",
337 dimensions=("instrument", "calibration_label"),
338 )
340 outputCrosstalk = cT.Output(
341 name="crosstalkProposal",
342 doc="Output proposed crosstalk calibration.",
343 storageClass="CrosstalkCalib",
344 dimensions=("instrument", "detector"),
345 multiple=False,
346 )
348 def __init__(self, *, config=None):
349 super().__init__(config=config)
351 if config.fluxOrder == 0:
352 self.inputs.discard("inputFluxes")
355class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig,
356 pipelineConnections=CrosstalkSolveConnections):
357 """Configuration for the solving of crosstalk from pixel ratios.
358 """
359 rejIter = Field(
360 dtype=int,
361 default=3,
362 doc="Number of rejection iterations for final coefficient calculation.",
363 )
364 rejSigma = Field(
365 dtype=float,
366 default=2.0,
367 doc="Rejection threshold (sigma) for final coefficient calculation.",
368 )
369 fluxOrder = Field(
370 dtype=int,
371 default=0,
372 doc="Polynomial order in source flux to fit crosstalk.",
373 )
374 doFiltering = Field(
375 dtype=bool,
376 default=False,
377 doc="Filter generated crosstalk to remove marginal measurements.",
378 )
381class CrosstalkSolveTask(pipeBase.PipelineTask,
382 pipeBase.CmdLineTask):
383 """Task to solve crosstalk from pixel ratios.
384 """
385 ConfigClass = CrosstalkSolveConfig
386 _DefaultName = 'cpCrosstalkSolve'
388 def runQuantum(self, butlerQC, inputRefs, outputRefs):
389 """Ensure that the input and output dimensions are passed along.
391 Parameters
392 ----------
393 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
394 Butler to operate on.
395 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
396 Input data refs to load.
397 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
398 Output data refs to persist.
399 """
400 inputs = butlerQC.get(inputRefs)
402 # Use the dimensions to set calib/provenance information.
403 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios]
404 inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
406 outputs = self.run(**inputs)
407 butlerQC.put(outputs, outputRefs)
409 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
410 """Combine ratios to produce crosstalk coefficients.
412 Parameters
413 ----------
414 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
415 A list of nested dictionaries of ratios indexed by target
416 and source chip, then by target and source amplifier.
417 inputFluxes : `list` [`dict` [`dict` [`list`]]]
418 A list of nested dictionaries of source pixel fluxes, indexed
419 by source chip and amplifier.
420 camera : `lsst.afw.cameraGeom.Camera`
421 Input camera.
422 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
423 DataIds to use to construct provenance.
424 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
425 DataIds to use to populate the output calibration.
427 Returns
428 -------
429 results : `lsst.pipe.base.Struct`
430 The results struct containing:
432 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
433 Final crosstalk calibration.
434 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
435 Provenance data for the new calibration.
437 Raises
438 ------
439 RuntimeError
440 Raised if the input data contains multiple target detectors.
442 Notes
443 -----
444 The lsstDebug.Info() method can be rewritten for __name__ =
445 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
447 debug.display['reduce'] : `bool`
448 Display a histogram of the combined ratio measurements for
449 a pair of source/target amplifiers from all input
450 exposures/detectors.
452 """
453 if outputDims:
454 calibChip = outputDims['detector']
455 instrument = outputDims['instrument']
456 else:
457 # calibChip needs to be set manually in Gen2.
458 calibChip = None
459 instrument = None
461 self.log.info("Combining measurements from %d ratios and %d fluxes",
462 len(inputRatios), len(inputFluxes) if inputFluxes else 0)
464 if inputFluxes is None:
465 inputFluxes = [None for exp in inputRatios]
467 combinedRatios = defaultdict(lambda: defaultdict(list))
468 combinedFluxes = defaultdict(lambda: defaultdict(list))
469 for ratioDict, fluxDict in zip(inputRatios, inputFluxes):
470 for targetChip in ratioDict:
471 if calibChip and targetChip != calibChip:
472 raise RuntimeError("Received multiple target chips!")
474 sourceChip = targetChip
475 if sourceChip in ratioDict[targetChip]:
476 ratios = ratioDict[targetChip][sourceChip]
478 for targetAmp in ratios:
479 for sourceAmp in ratios[targetAmp]:
480 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
481 if fluxDict:
482 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
483 # TODO: DM-21904
484 # Iterating over all other entries in ratioDict[targetChip] will yield
485 # inter-chip terms.
487 for targetAmp in combinedRatios:
488 for sourceAmp in combinedRatios[targetAmp]:
489 self.log.info("Read %d pixels for %s -> %s",
490 len(combinedRatios[targetAmp][sourceAmp]),
491 targetAmp, sourceAmp)
492 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
493 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp)
495 if self.config.fluxOrder == 0:
496 self.log.info("Fitting crosstalk coefficients.")
497 calib = self.measureCrosstalkCoefficients(combinedRatios,
498 self.config.rejIter, self.config.rejSigma)
499 else:
500 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.")
502 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid))
504 if self.config.doFiltering:
505 # This step will apply the calculated validity values to
506 # censor poorly measured coefficients.
507 self.log.info("Filtering measured crosstalk to remove invalid solutions.")
508 calib = self.filterCrosstalkCalib(calib)
510 # Populate the remainder of the calibration information.
511 calib.hasCrosstalk = True
512 calib.interChip = {}
513 calib._detectorName = calibChip
514 if camera:
515 for chip in camera:
516 if chip.getName() == calibChip:
517 calib._detectorSerial = chip.getSerial()
518 calib._instrument = instrument
519 calib.updateMetadata()
521 # Make an IsrProvenance().
522 provenance = IsrProvenance(calibType="CROSSTALK")
523 provenance._detectorName = calibChip
524 if inputDims:
525 provenance.fromDataIds(inputDims)
526 provenance._instrument = instrument
527 provenance.updateMetadata()
529 return pipeBase.Struct(
530 outputCrosstalk=calib,
531 outputProvenance=provenance,
532 )
534 def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma):
535 """Measure crosstalk coefficients from the ratios.
537 Given a list of ratios for each target/source amp combination,
538 we measure a sigma clipped mean and error.
540 The coefficient errors returned are the standard deviation of
541 the final set of clipped input ratios.
543 Parameters
544 ----------
545 ratios : `dict` of `dict` of `numpy.ndarray`
546 Catalog of arrays of ratios.
547 rejIter : `int`
548 Number of rejection iterations.
549 rejSigma : `float`
550 Rejection threshold (sigma).
552 Returns
553 -------
554 calib : `lsst.ip.isr.CrosstalkCalib`
555 The output crosstalk calibration.
557 Notes
558 -----
559 The lsstDebug.Info() method can be rewritten for __name__ =
560 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
562 debug.display['measure'] : `bool`
563 Display the CDF of the combined ratio measurements for
564 a pair of source/target amplifiers from the final set of
565 clipped input ratios.
566 """
567 calib = CrosstalkCalib(nAmp=len(ratios))
569 # Calibration stores coefficients as a numpy ndarray.
570 ordering = list(ratios.keys())
571 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
572 if ii == jj:
573 values = [0.0]
574 else:
575 values = np.array(ratios[ordering[ii]][ordering[jj]])
576 values = values[np.abs(values) < 1.0] # Discard unreasonable values
578 calib.coeffNum[ii][jj] = len(values)
580 if len(values) == 0:
581 self.log.warn("No values for matrix element %d,%d" % (ii, jj))
582 calib.coeffs[ii][jj] = np.nan
583 calib.coeffErr[ii][jj] = np.nan
584 calib.coeffValid[ii][jj] = False
585 else:
586 if ii != jj:
587 for rej in range(rejIter):
588 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
589 sigma = 0.741*(hi - lo)
590 good = np.abs(values - med) < rejSigma*sigma
591 if good.sum() == len(good):
592 break
593 values = values[good]
595 calib.coeffs[ii][jj] = np.mean(values)
596 if calib.coeffNum[ii][jj] == 1:
597 calib.coeffErr[ii][jj] = np.nan
598 else:
599 correctionFactor = self.sigmaClipCorrection(rejSigma)
600 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
601 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) >
602 calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
604 if calib.coeffNum[ii][jj] > 1:
605 self.debugRatios('measure', ratios, ordering[ii], ordering[jj],
606 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
608 return calib
610 @staticmethod
611 def sigmaClipCorrection(nSigClip):
612 """Correct measured sigma to account for clipping.
614 If we clip our input data and then measure sigma, then the
615 measured sigma is smaller than the true value because real
616 points beyond the clip threshold have been removed. This is a
617 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the
618 default parameters for measure crosstalk use nSigClip=2.0.
619 This causes the measured sigma to be about 15% smaller than
620 real. This formula corrects the issue, for the symmetric case
621 (upper clip threshold equal to lower clip threshold).
623 Parameters
624 ----------
625 nSigClip : `float`
626 Number of sigma the measurement was clipped by.
628 Returns
629 -------
630 scaleFactor : `float`
631 Scale factor to increase the measured sigma by.
633 """
634 varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip))
635 return 1.0 / np.sqrt(varFactor)
637 @staticmethod
638 def filterCrosstalkCalib(inCalib):
639 """Apply valid constraints to the measured values.
641 Any measured coefficient that is determined to be invalid is
642 set to zero, and has the error set to nan. The validation is
643 determined by checking that the measured coefficient is larger
644 than the calculated standard error of the mean.
646 Parameters
647 ----------
648 inCalib : `lsst.ip.isr.CrosstalkCalib`
649 Input calibration to filter.
651 Returns
652 -------
653 outCalib : `lsst.ip.isr.CrosstalkCalib`
654 Filtered calibration.
655 """
656 outCalib = CrosstalkCalib()
657 outCalib.numAmps = inCalib.numAmps
659 outCalib.coeffs = inCalib.coeffs
660 outCalib.coeffs[~inCalib.coeffValid] = 0.0
662 outCalib.coeffErr = inCalib.coeffErr
663 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
665 outCalib.coeffNum = inCalib.coeffNum
666 outCalib.coeffValid = inCalib.coeffValid
668 return outCalib
670 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
671 """Utility function to examine the final CT ratio set.
673 Parameters
674 ----------
675 stepname : `str`
676 State of processing to view.
677 ratios : `dict` of `dict` of `np.ndarray`
678 Array of measured CT ratios, indexed by source/victim
679 amplifier.
680 i : `str`
681 Index of the source amplifier.
682 j : `str`
683 Index of the target amplifier.
684 coeff : `float`, optional
685 Coefficient calculated to plot along with the simple mean.
686 valid : `bool`, optional
687 Validity to be added to the plot title.
688 """
689 frame = getDebugFrame(self._display, stepname)
690 if frame:
691 if i == j or ratios is None or len(ratios) < 1:
692 pass
694 ratioList = ratios[i][j]
695 if ratioList is None or len(ratioList) < 1:
696 pass
698 mean = np.mean(ratioList)
699 std = np.std(ratioList)
700 import matplotlib.pyplot as plt
701 figure = plt.figure(1)
702 figure.clear()
703 plt.hist(x=ratioList, bins=len(ratioList),
704 cumulative=True, color='b', density=True, histtype='step')
705 plt.xlabel("Measured pixel ratio")
706 plt.ylabel(f"CDF: n={len(ratioList)}")
707 plt.xlim(np.percentile(ratioList, [1.0, 99]))
708 plt.axvline(x=mean, color="k")
709 plt.axvline(x=coeff, color='g')
710 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r')
711 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r')
712 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
713 figure.show()
715 prompt = "Press Enter to continue: "
716 while True:
717 ans = input(prompt).lower()
718 if ans in ("", "c",):
719 break
720 elif ans in ("pdb", "p",):
721 import pdb
722 pdb.set_trace()
723 plt.close()
726class MeasureCrosstalkConfig(Config):
727 extract = ConfigurableField(
728 target=CrosstalkExtractTask,
729 doc="Task to measure pixel ratios.",
730 )
731 solver = ConfigurableField(
732 target=CrosstalkSolveTask,
733 doc="Task to convert ratio lists to crosstalk coefficients.",
734 )
737class MeasureCrosstalkTask(pipeBase.CmdLineTask):
738 """Measure intra-detector crosstalk.
740 See also
741 --------
742 lsst.ip.isr.crosstalk.CrosstalkCalib
743 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
744 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
746 Notes
747 -----
748 The crosstalk this method measures assumes that when a bright
749 pixel is found in one detector amplifier, all other detector
750 amplifiers may see a signal change in the same pixel location
751 (relative to the readout amplifier) as these other pixels are read
752 out at the same time.
754 After processing each input exposure through a limited set of ISR
755 stages, bright unmasked pixels above the threshold are identified.
756 The potential CT signal is found by taking the ratio of the
757 appropriate background-subtracted pixel value on the other
758 amplifiers to the input value on the source amplifier. If the
759 source amplifier has a large number of bright pixels as well, the
760 background level may be elevated, leading to poor ratio
761 measurements.
763 The set of ratios found between each pair of amplifiers across all
764 input exposures is then gathered to produce the final CT
765 coefficients. The sigma-clipped mean and sigma are returned from
766 these sets of ratios, with the coefficient to supply to the ISR
767 CrosstalkTask() being the multiplicative inverse of these values.
769 This Task simply calls the pipetask versions of the measure
770 crosstalk code.
771 """
772 ConfigClass = MeasureCrosstalkConfig
773 _DefaultName = "measureCrosstalk"
775 # Let's use this instead of messing with parseAndRun.
776 RunnerClass = DataRefListRunner
778 def __init__(self, **kwargs):
779 super().__init__(**kwargs)
780 self.makeSubtask("extract")
781 self.makeSubtask("solver")
783 def runDataRef(self, dataRefList):
784 """Run extract task on each of inputs in the dataRef list, then pass
785 that to the solver task.
787 Parameters
788 ----------
789 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
790 Data references for exposures for detectors to process.
792 Returns
793 -------
794 results : `lsst.pipe.base.Struct`
795 The results struct containing:
797 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
798 Final crosstalk calibration.
799 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
800 Provenance data for the new calibration.
802 Raises
803 ------
804 RuntimeError
805 Raised if multiple target detectors are supplied.
806 """
807 dataRef = dataRefList[0]
808 camera = dataRef.get("camera")
810 ratios = []
811 activeChip = None
812 for dataRef in dataRefList:
813 exposure = dataRef.get("postISRCCD")
814 if activeChip:
815 if exposure.getDetector().getName() != activeChip:
816 raise RuntimeError("Too many input detectors supplied!")
817 else:
818 activeChip = exposure.getDetector().getName()
820 self.extract.debugView("extract", exposure)
821 result = self.extract.run(exposure)
822 ratios.append(result.outputRatios)
824 finalResults = self.solver.run(ratios, camera=camera)
825 dataRef.put(finalResults.outputCrosstalk, "crosstalk")
827 return finalResults