Coverage for python/lsst/cp/pipe/measureCrosstalk.py: 18%
311 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-15 03:52 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-15 03:52 -0700
1# This file is part of cp_pipe
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import itertools
22import numpy as np
24from collections import defaultdict
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
29from lsstDebug import getDebugFrame
30from lsst.afw.detection import FootprintSet, Threshold
31from lsst.afw.display import getDisplay
32from lsst.pex.config import Config, Field, ListField, ConfigurableField
33from lsst.ip.isr import CrosstalkCalib, IsrProvenance
34from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
35from lsst.cp.pipe.utils import (ddict2dict, sigmaClipCorrection)
37from ._lookupStaticCalibration import lookupStaticCalibration
39__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask",
40 "CrosstalkSolveTask", "CrosstalkSolveConfig",
41 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"]
44class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections,
45 dimensions=("instrument", "exposure", "detector")):
46 inputExp = cT.Input(
47 name="crosstalkInputs",
48 doc="Input post-ISR processed exposure to measure crosstalk from.",
49 storageClass="Exposure",
50 dimensions=("instrument", "exposure", "detector"),
51 multiple=False,
52 )
53 # TODO: Depends on DM-21904.
54 sourceExp = cT.Input(
55 name="crosstalkSource",
56 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
57 storageClass="Exposure",
58 dimensions=("instrument", "exposure", "detector"),
59 multiple=True,
60 deferLoad=True,
61 # lookupFunction=None,
62 )
64 outputRatios = cT.Output(
65 name="crosstalkRatios",
66 doc="Extracted crosstalk pixel ratios.",
67 storageClass="StructuredDataDict",
68 dimensions=("instrument", "exposure", "detector"),
69 )
70 outputFluxes = cT.Output(
71 name="crosstalkFluxes",
72 doc="Source pixel fluxes used in ratios.",
73 storageClass="StructuredDataDict",
74 dimensions=("instrument", "exposure", "detector"),
75 )
77 def __init__(self, *, config=None):
78 super().__init__(config=config)
79 # Discard sourceExp until DM-21904 allows full interchip
80 # measurements.
81 self.inputs.discard("sourceExp")
84class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=CrosstalkExtractConnections):
86 """Configuration for the measurement of pixel ratios.
87 """
89 doMeasureInterchip = Field(
90 dtype=bool,
91 default=False,
92 doc="Measure inter-chip crosstalk as well?",
93 )
94 threshold = Field(
95 dtype=float,
96 default=30000,
97 doc="Minimum level of source pixels for which to measure crosstalk."
98 )
99 ignoreSaturatedPixels = Field(
100 dtype=bool,
101 default=False,
102 doc="Should saturated pixels be ignored?"
103 )
104 badMask = ListField(
105 dtype=str,
106 default=["BAD", "INTRP"],
107 doc="Mask planes to ignore when identifying source pixels."
108 )
109 isTrimmed = Field(
110 dtype=bool,
111 default=True,
112 doc="Is the input exposure trimmed?"
113 )
115 def validate(self):
116 super().validate()
118 # Ensure the handling of the SAT mask plane is consistent
119 # with the ignoreSaturatedPixels value.
120 if self.ignoreSaturatedPixels:
121 if 'SAT' not in self.badMask:
122 self.badMask.append('SAT')
123 else:
124 if 'SAT' in self.badMask:
125 self.badMask = [mask for mask in self.badMask if mask != 'SAT']
128class CrosstalkExtractTask(pipeBase.PipelineTask,
129 pipeBase.CmdLineTask):
130 """Task to measure pixel ratios to find crosstalk.
131 """
133 ConfigClass = CrosstalkExtractConfig
134 _DefaultName = 'cpCrosstalkExtract'
136 def run(self, inputExp, sourceExps=[]):
137 """Measure pixel ratios between amplifiers in inputExp.
139 Extract crosstalk ratios between different amplifiers.
141 For pixels above ``config.threshold``, we calculate the ratio
142 between each background-subtracted target amp and the source
143 amp. We return a list of ratios for each pixel for each
144 target/source combination, as nested dictionary containing the
145 ratio.
147 Parameters
148 ----------
149 inputExp : `lsst.afw.image.Exposure`
150 Input exposure to measure pixel ratios on.
151 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
152 List of chips to use as sources to measure inter-chip
153 crosstalk.
155 Returns
156 -------
157 results : `lsst.pipe.base.Struct`
158 The results struct containing:
160 ``outputRatios``
161 A catalog of ratio lists. The dictionaries are
162 indexed such that:
163 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
164 contains the ratio list for that combination (`dict`
165 [`dict` [`dict` [`dict` [`list`]]]]).
166 ``outputFluxes``
167 A catalog of flux lists. The dictionaries are
168 indexed such that:
169 outputFluxes[sourceChip][sourceAmp] contains the flux
170 list used in the outputRatios (`dict` [`dict`
171 [`list`]]).
172 """
173 outputRatios = defaultdict(lambda: defaultdict(dict))
174 outputFluxes = defaultdict(lambda: defaultdict(dict))
176 threshold = self.config.threshold
177 badPixels = list(self.config.badMask)
179 targetDetector = inputExp.getDetector()
180 targetChip = targetDetector.getName()
182 # Always look at the target chip first, then go to any other
183 # supplied exposures.
184 sourceExtractExps = [inputExp]
185 sourceExtractExps.extend(sourceExps)
187 self.log.info("Measuring full detector background for target: %s", targetChip)
188 targetIm = inputExp.getMaskedImage()
189 FootprintSet(targetIm, Threshold(threshold), "DETECTED")
190 detected = targetIm.getMask().getPlaneBitMask("DETECTED")
191 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
193 self.debugView('extract', inputExp)
195 for sourceExp in sourceExtractExps:
196 sourceDetector = sourceExp.getDetector()
197 sourceChip = sourceDetector.getName()
198 sourceIm = sourceExp.getMaskedImage()
199 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
200 self.log.info("Measuring crosstalk from source: %s", sourceChip)
202 if sourceExp != inputExp:
203 FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
204 detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
206 # The dictionary of amp-to-amp ratios for this pair of
207 # source->target detectors.
208 ratioDict = defaultdict(lambda: defaultdict(list))
209 extractedCount = 0
211 for sourceAmp in sourceDetector:
212 sourceAmpName = sourceAmp.getName()
213 sourceAmpBBox = sourceAmp.getBBox() if self.config.isTrimmed else sourceAmp.getRawDataBBox()
214 sourceAmpImage = sourceIm[sourceAmpBBox]
215 sourceMask = sourceAmpImage.mask.array
216 select = ((sourceMask & detected > 0)
217 & (sourceMask & bad == 0)
218 & np.isfinite(sourceAmpImage.image.array))
219 count = np.sum(select)
220 self.log.debug(" Source amplifier: %s", sourceAmpName)
222 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
224 for targetAmp in targetDetector:
225 # iterate over targetExposure
226 targetAmpName = targetAmp.getName()
227 if sourceAmpName == targetAmpName and sourceChip == targetChip:
228 ratioDict[targetAmpName][sourceAmpName] = []
229 continue
230 self.log.debug(" Target amplifier: %s", targetAmpName)
232 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
233 targetAmp, sourceAmp,
234 isTrimmed=self.config.isTrimmed)
235 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
236 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
237 extractedCount += count
239 self.debugPixels('pixels',
240 sourceAmpImage.image.array[select],
241 targetAmpImage.array[select] - bg,
242 sourceAmpName, targetAmpName)
244 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
245 extractedCount, sourceChip, targetChip, bg)
246 outputRatios[targetChip][sourceChip] = ratioDict
248 return pipeBase.Struct(
249 outputRatios=ddict2dict(outputRatios),
250 outputFluxes=ddict2dict(outputFluxes)
251 )
253 def debugView(self, stepname, exposure):
254 """Utility function to examine the image being processed.
256 Parameters
257 ----------
258 stepname : `str`
259 State of processing to view.
260 exposure : `lsst.afw.image.Exposure`
261 Exposure to view.
262 """
263 frame = getDebugFrame(self._display, stepname)
264 if frame:
265 display = getDisplay(frame)
266 display.scale('asinh', 'zscale')
267 display.mtv(exposure)
269 prompt = "Press Enter to continue: "
270 while True:
271 ans = input(prompt).lower()
272 if ans in ("", "c",):
273 break
275 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
276 """Utility function to examine the CT ratio pixel values.
278 Parameters
279 ----------
280 stepname : `str`
281 State of processing to view.
282 pixelsIn : `np.ndarray`, (N,)
283 Pixel values from the potential crosstalk source.
284 pixelsOut : `np.ndarray`, (N,)
285 Pixel values from the potential crosstalk target.
286 sourceName : `str`
287 Source amplifier name
288 targetName : `str`
289 Target amplifier name
290 """
291 frame = getDebugFrame(self._display, stepname)
292 if frame:
293 import matplotlib.pyplot as plt
294 figure = plt.figure(1)
295 figure.clear()
297 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
298 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+')
299 plt.xlabel("Source amplifier pixel value")
300 plt.ylabel("Measured pixel ratio")
301 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: "
302 f"{(np.median(pixelsOut / pixelsIn))}")
303 figure.show()
305 prompt = "Press Enter to continue: "
306 while True:
307 ans = input(prompt).lower()
308 if ans in ("", "c",):
309 break
310 plt.close()
313class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections,
314 dimensions=("instrument", "detector")):
315 inputRatios = cT.Input(
316 name="crosstalkRatios",
317 doc="Ratios measured for an input exposure.",
318 storageClass="StructuredDataDict",
319 dimensions=("instrument", "exposure", "detector"),
320 multiple=True,
321 )
322 inputFluxes = cT.Input(
323 name="crosstalkFluxes",
324 doc="Fluxes of CT source pixels, for nonlinear fits.",
325 storageClass="StructuredDataDict",
326 dimensions=("instrument", "exposure", "detector"),
327 multiple=True,
328 )
329 camera = cT.PrerequisiteInput(
330 name="camera",
331 doc="Camera the input data comes from.",
332 storageClass="Camera",
333 dimensions=("instrument",),
334 isCalibration=True,
335 lookupFunction=lookupStaticCalibration,
336 )
338 outputCrosstalk = cT.Output(
339 name="crosstalk",
340 doc="Output proposed crosstalk calibration.",
341 storageClass="CrosstalkCalib",
342 dimensions=("instrument", "detector"),
343 multiple=False,
344 isCalibration=True,
345 )
347 def __init__(self, *, config=None):
348 super().__init__(config=config)
350 if config.fluxOrder == 0:
351 self.inputs.discard("inputFluxes")
354class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig,
355 pipelineConnections=CrosstalkSolveConnections):
356 """Configuration for the solving of crosstalk from pixel ratios.
357 """
359 rejIter = Field(
360 dtype=int,
361 default=3,
362 doc="Number of rejection iterations for final coefficient calculation.",
363 )
364 rejSigma = Field(
365 dtype=float,
366 default=2.0,
367 doc="Rejection threshold (sigma) for final coefficient calculation.",
368 )
369 fluxOrder = Field(
370 dtype=int,
371 default=0,
372 doc="Polynomial order in source flux to fit crosstalk.",
373 )
374 significanceLimit = Field(
375 dtype=float,
376 default=3.0,
377 doc="Sigma significance level to use in marking a coefficient valid.",
378 )
379 doSignificanceScaling = Field(
380 dtype=bool,
381 default=True,
382 doc="Scale error by 1/sqrt(N) in calculating significant coefficients?",
383 )
384 doFiltering = Field(
385 dtype=bool,
386 default=False,
387 doc="Filter generated crosstalk to remove marginal measurements?",
388 )
391class CrosstalkSolveTask(pipeBase.PipelineTask,
392 pipeBase.CmdLineTask):
393 """Task to solve crosstalk from pixel ratios.
394 """
396 ConfigClass = CrosstalkSolveConfig
397 _DefaultName = 'cpCrosstalkSolve'
399 def runQuantum(self, butlerQC, inputRefs, outputRefs):
400 """Ensure that the input and output dimensions are passed along.
402 Parameters
403 ----------
404 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
405 Butler to operate on.
406 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
407 Input data refs to load.
408 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
409 Output data refs to persist.
410 """
411 inputs = butlerQC.get(inputRefs)
413 # Use the dimensions to set calib/provenance information.
414 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios]
415 inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
417 outputs = self.run(**inputs)
418 butlerQC.put(outputs, outputRefs)
420 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
421 """Combine ratios to produce crosstalk coefficients.
423 Parameters
424 ----------
425 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
426 A list of nested dictionaries of ratios indexed by target
427 and source chip, then by target and source amplifier.
428 inputFluxes : `list` [`dict` [`dict` [`list`]]]
429 A list of nested dictionaries of source pixel fluxes, indexed
430 by source chip and amplifier.
431 camera : `lsst.afw.cameraGeom.Camera`
432 Input camera.
433 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
434 DataIds to use to construct provenance.
435 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
436 DataIds to use to populate the output calibration.
438 Returns
439 -------
440 results : `lsst.pipe.base.Struct`
441 The results struct containing:
443 ``outputCrosstalk``
444 Final crosstalk calibration
445 (`lsst.ip.isr.CrosstalkCalib`).
446 ``outputProvenance``
447 Provenance data for the new calibration
448 (`lsst.ip.isr.IsrProvenance`).
450 Raises
451 ------
452 RuntimeError
453 Raised if the input data contains multiple target detectors.
454 """
455 if outputDims:
456 calibChip = outputDims['detector']
457 instrument = outputDims['instrument']
458 else:
459 # calibChip needs to be set manually in Gen2.
460 calibChip = None
461 instrument = None
463 if camera and calibChip is not None:
464 calibDetector = camera[calibChip]
465 ordering = [amp.getName() for amp in calibDetector]
466 else:
467 calibDetector = None
468 ordering = None
470 self.log.info("Combining measurements from %d ratios and %d fluxes",
471 len(inputRatios), len(inputFluxes) if inputFluxes else 0)
473 if inputFluxes is None:
474 inputFluxes = [None for exp in inputRatios]
476 combinedRatios = defaultdict(lambda: defaultdict(list))
477 combinedFluxes = defaultdict(lambda: defaultdict(list))
478 for ratioDict, fluxDict in zip(inputRatios, inputFluxes):
479 for targetChip in ratioDict:
480 if calibChip and targetChip != calibChip and targetChip != calibDetector.getName():
481 raise RuntimeError(f"Target chip: {targetChip} does not match calibration dimension: "
482 f"{calibChip}, {calibDetector.getName()}!")
484 sourceChip = targetChip
485 if sourceChip in ratioDict[targetChip]:
486 ratios = ratioDict[targetChip][sourceChip]
488 for targetAmp in ratios:
489 for sourceAmp in ratios[targetAmp]:
490 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
491 if fluxDict:
492 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
493 # TODO: DM-21904
494 # Iterating over all other entries in
495 # ratioDict[targetChip] will yield inter-chip terms.
497 for targetAmp in combinedRatios:
498 for sourceAmp in combinedRatios[targetAmp]:
499 self.log.info("Read %d pixels for %s -> %s",
500 len(combinedRatios[targetAmp][sourceAmp]),
501 targetAmp, sourceAmp)
502 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
503 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp)
505 if self.config.fluxOrder == 0:
506 self.log.info("Fitting crosstalk coefficients.")
508 calib = self.measureCrosstalkCoefficients(combinedRatios, ordering,
509 self.config.rejIter, self.config.rejSigma)
510 else:
511 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.")
513 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid))
515 if self.config.doFiltering:
516 # This step will apply the calculated validity values to
517 # censor poorly measured coefficients.
518 self.log.info("Filtering measured crosstalk to remove invalid solutions.")
519 calib = self.filterCrosstalkCalib(calib)
521 # Populate the remainder of the calibration information.
522 calib.hasCrosstalk = True
523 calib.interChip = {}
525 # calibChip is the detector dimension, which is the detector Id
526 calib._detectorId = calibChip
527 if calibDetector:
528 calib._detectorName = calibDetector.getName()
529 calib._detectorSerial = calibDetector.getSerial()
531 calib._instrument = instrument
532 calib.updateMetadata(setCalibId=True, setDate=True)
534 # Make an IsrProvenance().
535 provenance = IsrProvenance(calibType="CROSSTALK")
536 provenance._detectorName = calibChip
537 if inputDims:
538 provenance.fromDataIds(inputDims)
539 provenance._instrument = instrument
540 provenance.updateMetadata()
542 return pipeBase.Struct(
543 outputCrosstalk=calib,
544 outputProvenance=provenance,
545 )
547 def measureCrosstalkCoefficients(self, ratios, ordering, rejIter, rejSigma):
548 """Measure crosstalk coefficients from the ratios.
550 Given a list of ratios for each target/source amp combination,
551 we measure a sigma clipped mean and error.
553 The coefficient errors returned are the standard deviation of
554 the final set of clipped input ratios.
556 Parameters
557 ----------
558 ratios : `dict` [`dict` [`numpy.ndarray`]]
559 Catalog of arrays of ratios. The ratio arrays are one-dimensional
560 ordering : `list` [`str`] or None
561 List to use as a mapping between amplifier names (the
562 elements of the list) and their position in the output
563 calibration (the matching index of the list). If no
564 ordering is supplied, the order of the keys in the ratio
565 catalog is used.
566 rejIter : `int`
567 Number of rejection iterations.
568 rejSigma : `float`
569 Rejection threshold (sigma).
571 Returns
572 -------
573 calib : `lsst.ip.isr.CrosstalkCalib`
574 The output crosstalk calibration.
575 """
576 calib = CrosstalkCalib(nAmp=len(ratios))
578 if ordering is None:
579 ordering = list(ratios.keys())
581 # Calibration stores coefficients as a numpy ndarray.
582 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
583 if ii == jj:
584 values = [0.0]
585 else:
586 values = np.array(ratios[ordering[ii]][ordering[jj]])
587 values = values[np.abs(values) < 1.0] # Discard unreasonable values
589 # Sigma clip using the inter-quartile distance and a
590 # normal distribution.
591 if ii != jj:
592 for rej in range(rejIter):
593 if len(values) == 0:
594 break
595 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
596 sigma = 0.741*(hi - lo)
597 good = np.abs(values - med) < rejSigma*sigma
598 if good.sum() == len(good) or good.sum() == 0:
599 break
600 values = values[good]
602 calib.coeffNum[ii][jj] = len(values)
603 significanceThreshold = 0.0
604 if len(values) == 0:
605 self.log.warning("No values for matrix element %d,%d" % (ii, jj))
606 calib.coeffs[ii][jj] = np.nan
607 calib.coeffErr[ii][jj] = np.nan
608 calib.coeffValid[ii][jj] = False
609 else:
610 calib.coeffs[ii][jj] = np.mean(values)
611 if calib.coeffNum[ii][jj] == 1:
612 calib.coeffErr[ii][jj] = np.nan
613 calib.coeffValid[ii][jj] = False
614 else:
615 correctionFactor = sigmaClipCorrection(rejSigma)
616 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
618 # Use sample stdev.
619 significanceThreshold = self.config.significanceLimit * calib.coeffErr[ii][jj]
620 if self.config.doSignificanceScaling is True:
621 # Enabling this calculates the stdev of the mean.
622 significanceThreshold /= np.sqrt(calib.coeffNum[ii][jj])
623 calib.coeffValid[ii][jj] = np.abs(calib.coeffs[ii][jj]) > significanceThreshold
624 self.debugRatios('measure', ratios, ordering[ii], ordering[jj],
625 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
626 self.log.info("Measured %s -> %s Coeff: %e Err: %e N: %d Valid: %s Limit: %e",
627 ordering[jj], ordering[ii], calib.coeffs[ii][jj], calib.coeffErr[ii][jj],
628 calib.coeffNum[ii][jj], calib.coeffValid[ii][jj], significanceThreshold)
630 return calib
632 @staticmethod
633 def filterCrosstalkCalib(inCalib):
634 """Apply valid constraints to the measured values.
636 Any measured coefficient that is determined to be invalid is
637 set to zero, and has the error set to nan. The validation is
638 determined by checking that the measured coefficient is larger
639 than the calculated standard error of the mean.
641 Parameters
642 ----------
643 inCalib : `lsst.ip.isr.CrosstalkCalib`
644 Input calibration to filter.
646 Returns
647 -------
648 outCalib : `lsst.ip.isr.CrosstalkCalib`
649 Filtered calibration.
650 """
651 outCalib = CrosstalkCalib()
652 outCalib.numAmps = inCalib.numAmps
654 outCalib.coeffs = inCalib.coeffs
655 outCalib.coeffs[~inCalib.coeffValid] = 0.0
657 outCalib.coeffErr = inCalib.coeffErr
658 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
660 outCalib.coeffNum = inCalib.coeffNum
661 outCalib.coeffValid = inCalib.coeffValid
663 return outCalib
665 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
666 """Utility function to examine the final CT ratio set.
668 Parameters
669 ----------
670 stepname : `str`
671 State of processing to view.
672 ratios : `dict` [`dict` [`numpy.ndarray`]]
673 Array of measured CT ratios, indexed by source/victim
674 amplifier. These arrays are one-dimensional.
675 i : `str`
676 Index of the source amplifier.
677 j : `str`
678 Index of the target amplifier.
679 coeff : `float`, optional
680 Coefficient calculated to plot along with the simple mean.
681 valid : `bool`, optional
682 Validity to be added to the plot title.
683 """
684 frame = getDebugFrame(self._display, stepname)
685 if frame:
686 if i == j or ratios is None or len(ratios) < 1:
687 pass
689 ratioList = ratios[i][j]
690 if ratioList is None or len(ratioList) < 1:
691 pass
693 mean = np.mean(ratioList)
694 std = np.std(ratioList)
695 import matplotlib.pyplot as plt
696 figure = plt.figure(1)
697 figure.clear()
698 plt.hist(x=ratioList, bins=len(ratioList),
699 cumulative=True, color='b', density=True, histtype='step')
700 plt.xlabel("Measured pixel ratio")
701 plt.ylabel(f"CDF: n={len(ratioList)}")
702 plt.xlim(np.percentile(ratioList, [1.0, 99]))
703 plt.axvline(x=mean, color="k")
704 plt.axvline(x=coeff, color='g')
705 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r')
706 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r')
707 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
708 figure.show()
710 prompt = "Press Enter to continue: "
711 while True:
712 ans = input(prompt).lower()
713 if ans in ("", "c",):
714 break
715 elif ans in ("pdb", "p",):
716 import pdb
717 pdb.set_trace()
718 plt.close()
721class MeasureCrosstalkConfig(Config):
722 extract = ConfigurableField(
723 target=CrosstalkExtractTask,
724 doc="Task to measure pixel ratios.",
725 )
726 solver = ConfigurableField(
727 target=CrosstalkSolveTask,
728 doc="Task to convert ratio lists to crosstalk coefficients.",
729 )
732class MeasureCrosstalkTask(pipeBase.CmdLineTask):
733 """Measure intra-detector crosstalk.
735 See also
736 --------
737 lsst.ip.isr.crosstalk.CrosstalkCalib
738 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
739 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
741 Notes
742 -----
743 The crosstalk this method measures assumes that when a bright
744 pixel is found in one detector amplifier, all other detector
745 amplifiers may see a signal change in the same pixel location
746 (relative to the readout amplifier) as these other pixels are read
747 out at the same time.
749 After processing each input exposure through a limited set of ISR
750 stages, bright unmasked pixels above the threshold are identified.
751 The potential CT signal is found by taking the ratio of the
752 appropriate background-subtracted pixel value on the other
753 amplifiers to the input value on the source amplifier. If the
754 source amplifier has a large number of bright pixels as well, the
755 background level may be elevated, leading to poor ratio
756 measurements.
758 The set of ratios found between each pair of amplifiers across all
759 input exposures is then gathered to produce the final CT
760 coefficients. The sigma-clipped mean and sigma are returned from
761 these sets of ratios, with the coefficient to supply to the ISR
762 CrosstalkTask() being the multiplicative inverse of these values.
764 This Task simply calls the pipetask versions of the measure
765 crosstalk code.
766 """
768 ConfigClass = MeasureCrosstalkConfig
769 _DefaultName = "measureCrosstalk"
771 # Let's use this instead of messing with parseAndRun.
772 RunnerClass = DataRefListRunner
774 def __init__(self, **kwargs):
775 super().__init__(**kwargs)
776 self.makeSubtask("extract")
777 self.makeSubtask("solver")
779 def runDataRef(self, dataRefList):
780 """Run extract task on each of inputs in the dataRef list, then pass
781 that to the solver task.
783 Parameters
784 ----------
785 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
786 Data references for exposures for detectors to process.
788 Returns
789 -------
790 results : `lsst.pipe.base.Struct`
791 The results struct containing:
793 ``outputCrosstalk``
794 Final crosstalk calibration
795 (`lsst.ip.isr.CrosstalkCalib`).
796 ``outputProvenance``
797 Provenance data for the new calibration
798 (`lsst.ip.isr.IsrProvenance`).
800 Raises
801 ------
802 RuntimeError
803 Raised if multiple target detectors are supplied.
804 """
805 dataRef = dataRefList[0]
806 camera = dataRef.get("camera")
808 ratios = []
809 activeChip = None
810 for dataRef in dataRefList:
811 exposure = dataRef.get("postISRCCD")
812 if activeChip:
813 if exposure.getDetector().getName() != activeChip:
814 raise RuntimeError("Too many input detectors supplied!")
815 else:
816 activeChip = exposure.getDetector().getName()
818 self.extract.debugView("extract", exposure)
819 result = self.extract.run(exposure)
820 ratios.append(result.outputRatios)
822 for detIter, detector in enumerate(camera):
823 if detector.getName() == activeChip:
824 detectorId = detIter
825 outputDims = {'instrument': camera.getName(),
826 'detector': detectorId,
827 }
829 finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims)
830 dataRef.put(finalResults.outputCrosstalk, "crosstalk")
832 return finalResults