23 from scipy.stats
import norm
25 from collections
import defaultdict
30 from lsstDebug
import getDebugFrame
34 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
35 from lsst.pipe.tasks.getRepositoryData
import DataRefListRunner
37 __all__ = [
"CrosstalkExtractConfig",
"CrosstalkExtractTask",
38 "CrosstalkSolveTask",
"CrosstalkSolveConfig",
39 "MeasureCrosstalkConfig",
"MeasureCrosstalkTask"]
43 dimensions=(
"instrument",
"exposure",
"detector")):
45 name=
"crosstalkInputs",
46 doc=
"Input post-ISR processed exposure to measure crosstalk from.",
47 storageClass=
"Exposure",
48 dimensions=(
"instrument",
"exposure",
"detector"),
53 name=
"crosstalkSource",
54 doc=
"Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
55 storageClass=
"Exposure",
56 dimensions=(
"instrument",
"exposure",
"detector"),
62 outputRatios = cT.Output(
63 name=
"crosstalkRatios",
64 doc=
"Extracted crosstalk pixel ratios.",
65 storageClass=
"StructuredDataDict",
66 dimensions=(
"instrument",
"exposure",
"detector"),
68 outputFluxes = cT.Output(
69 name=
"crosstalkFluxes",
70 doc=
"Source pixel fluxes used in ratios.",
71 storageClass=
"StructuredDataDict",
72 dimensions=(
"instrument",
"exposure",
"detector"),
79 self.inputs.discard(
"sourceExp")
83 pipelineConnections=CrosstalkExtractConnections):
84 """Configuration for the measurement of pixel ratios.
86 doMeasureInterchip = Field(
89 doc=
"Measure inter-chip crosstalk as well?",
94 doc=
"Minimum level of source pixels for which to measure crosstalk."
96 ignoreSaturatedPixels = Field(
99 doc=
"Should saturated pixels be ignored?"
103 default=[
"BAD",
"INTRP"],
104 doc=
"Mask planes to ignore when identifying source pixels."
109 doc=
"Is the input exposure trimmed?"
126 pipeBase.CmdLineTask):
127 """Task to measure pixel ratios to find crosstalk.
129 ConfigClass = CrosstalkExtractConfig
130 _DefaultName =
'cpCrosstalkExtract'
132 def run(self, inputExp, sourceExps=[]):
133 """Measure pixel ratios between amplifiers in inputExp.
135 Extract crosstalk ratios between different amplifiers.
137 For pixels above ``config.threshold``, we calculate the ratio
138 between each background-subtracted target amp and the source
139 amp. We return a list of ratios for each pixel for each
140 target/source combination, as nested dictionary containing the
145 inputExp : `lsst.afw.image.Exposure`
146 Input exposure to measure pixel ratios on.
147 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
148 List of chips to use as sources to measure inter-chip
153 results : `lsst.pipe.base.Struct`
154 The results struct containing:
156 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
157 A catalog of ratio lists. The dictionaries are
159 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
160 contains the ratio list for that combination.
161 ``outputFluxes`` : `dict` [`dict` [`list`]]
162 A catalog of flux lists. The dictionaries are
164 outputFluxes[sourceChip][sourceAmp]
165 contains the flux list used in the outputRatios.
169 The lsstDebug.Info() method can be rewritten for __name__ =
170 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
172 debug.display['extract'] : `bool`
173 Display the exposure under consideration, with the pixels used
174 for crosstalk measurement indicated by the DETECTED mask plane.
175 debug.display['pixels'] : `bool`
176 Display a plot of the ratio calculated for each pixel used in this
177 exposure, split by amplifier pairs. The median value is listed
180 outputRatios = defaultdict(
lambda: defaultdict(dict))
181 outputFluxes = defaultdict(
lambda: defaultdict(dict))
183 threshold = self.config.threshold
184 badPixels = list(self.config.badMask)
186 targetDetector = inputExp.getDetector()
187 targetChip = targetDetector.getName()
190 sourceExtractExps = [inputExp]
191 sourceExtractExps.extend(sourceExps)
193 self.log.info(
"Measuring full detector background for target: %s", targetChip)
194 targetIm = inputExp.getMaskedImage()
196 detected = targetIm.getMask().getPlaneBitMask(
"DETECTED")
197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + [
"DETECTED"])
201 for sourceExp
in sourceExtractExps:
202 sourceDetector = sourceExp.getDetector()
203 sourceChip = sourceDetector.getName()
204 sourceIm = sourceExp.getMaskedImage()
205 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
206 self.log.info(
"Measuring crosstalk from source: %s", sourceChip)
208 if sourceExp != inputExp:
210 detected = sourceIm.getMask().getPlaneBitMask(
"DETECTED")
213 ratioDict = defaultdict(
lambda: defaultdict(list))
216 for sourceAmp
in sourceDetector:
217 sourceAmpName = sourceAmp.getName()
218 sourceAmpImage = sourceIm[sourceAmp.getBBox()]
219 sourceMask = sourceAmpImage.mask.array
220 select = ((sourceMask & detected > 0) &
221 (sourceMask & bad == 0) &
222 np.isfinite(sourceAmpImage.image.array))
223 count = np.sum(select)
224 self.log.debug(
" Source amplifier: %s", sourceAmpName)
226 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select]
228 for targetAmp
in targetDetector:
230 targetAmpName = targetAmp.getName()
231 if sourceAmpName == targetAmpName
and sourceChip == targetChip:
232 ratioDict[sourceAmpName][targetAmpName] = []
234 self.log.debug(
" Target amplifier: %s", targetAmpName)
236 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
237 targetAmp, sourceAmp,
238 isTrimmed=self.config.isTrimmed)
239 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
240 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
241 extractedCount += count
244 sourceAmpImage.image.array[select],
245 targetAmpImage.array[select] - bg,
246 sourceAmpName, targetAmpName)
248 self.log.info(
"Extracted %d pixels from %s -> %s (targetBG: %f)",
249 extractedCount, sourceChip, targetChip, bg)
250 outputRatios[targetChip][sourceChip] = ratioDict
252 return pipeBase.Struct(
253 outputRatios=outputRatios,
254 outputFluxes=outputFluxes
258 """Utility function to examine the image being processed.
263 State of processing to view.
264 exposure : `lsst.afw.image.Exposure`
267 frame = getDebugFrame(self._display, stepname)
269 display = getDisplay(frame)
270 display.scale(
'asinh',
'zscale')
271 display.mtv(exposure)
273 prompt =
"Press Enter to continue: "
275 ans = input(prompt).lower()
276 if ans
in (
"",
"c",):
279 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
280 """Utility function to examine the CT ratio pixel values.
285 State of processing to view.
286 pixelsIn : `np.ndarray`
287 Pixel values from the potential crosstalk source.
288 pixelsOut : `np.ndarray`
289 Pixel values from the potential crosstalk target.
291 Source amplifier name
293 Target amplifier name
295 frame = getDebugFrame(self._display, stepname)
297 import matplotlib.pyplot
as plt
298 figure = plt.figure(1)
301 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
302 axes.plt(pixelsIn, pixelsOut / pixelsIn,
'k+')
303 plt.xlabel(
"Source amplifier pixel value")
304 plt.ylabel(
"Measured pixel ratio")
305 plt.title(f
"(Source {sourceName} -> Target {targetName}) median ratio: "
306 f
"{(np.median(pixelsOut / pixelsIn))}")
309 prompt =
"Press Enter to continue: "
311 ans = input(prompt).lower()
312 if ans
in (
"",
"c",):
318 dimensions=(
"instrument",
"detector")):
319 inputRatios = cT.Input(
320 name=
"crosstalkRatios",
321 doc=
"Ratios measured for an input exposure.",
322 storageClass=
"StructuredDataDict",
323 dimensions=(
"instrument",
"exposure",
"detector"),
326 inputFluxes = cT.Input(
327 name=
"crosstalkFluxes",
328 doc=
"Fluxes of CT source pixels, for nonlinear fits.",
329 storageClass=
"StructuredDataDict",
330 dimensions=(
"instrument",
"exposure",
"detector"),
333 camera = cT.PrerequisiteInput(
335 doc=
"Camera the input data comes from.",
336 storageClass=
"Camera",
337 dimensions=(
"instrument",
"calibration_label"),
340 outputCrosstalk = cT.Output(
341 name=
"crosstalkProposal",
342 doc=
"Output proposed crosstalk calibration.",
343 storageClass=
"CrosstalkCalib",
344 dimensions=(
"instrument",
"detector"),
351 if config.fluxOrder == 0:
352 self.inputs.discard(
"inputFluxes")
356 pipelineConnections=CrosstalkSolveConnections):
357 """Configuration for the solving of crosstalk from pixel ratios.
362 doc=
"Number of rejection iterations for final coefficient calculation.",
367 doc=
"Rejection threshold (sigma) for final coefficient calculation.",
372 doc=
"Polynomial order in source flux to fit crosstalk.",
377 doc=
"Filter generated crosstalk to remove marginal measurements.",
382 pipeBase.CmdLineTask):
383 """Task to solve crosstalk from pixel ratios.
385 ConfigClass = CrosstalkSolveConfig
386 _DefaultName =
'cpCrosstalkSolve'
389 """Ensure that the input and output dimensions are passed along.
393 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
394 Butler to operate on.
395 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
396 Input data refs to load.
397 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
398 Output data refs to persist.
400 inputs = butlerQC.get(inputRefs)
403 inputs[
'inputDims'] = [exp.dataId.byName()
for exp
in inputRefs.inputRatios]
404 inputs[
'outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
406 outputs = self.
run(**inputs)
407 butlerQC.put(outputs, outputRefs)
409 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
410 """Combine ratios to produce crosstalk coefficients.
414 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
415 A list of nested dictionaries of ratios indexed by target
416 and source chip, then by target and source amplifier.
417 inputFluxes : `list` [`dict` [`dict` [`list`]]]
418 A list of nested dictionaries of source pixel fluxes, indexed
419 by source chip and amplifier.
420 camera : `lsst.afw.cameraGeom.Camera`
422 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
423 DataIds to use to construct provenance.
424 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
425 DataIds to use to populate the output calibration.
429 results : `lsst.pipe.base.Struct`
430 The results struct containing:
432 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
433 Final crosstalk calibration.
434 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
435 Provenance data for the new calibration.
440 Raised if the input data contains multiple target detectors.
444 The lsstDebug.Info() method can be rewritten for __name__ =
445 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
447 debug.display['reduce'] : `bool`
448 Display a histogram of the combined ratio measurements for
449 a pair of source/target amplifiers from all input
454 calibChip = outputDims[
'detector']
455 instrument = outputDims[
'instrument']
461 self.log.info(
"Combining measurements from %d ratios and %d fluxes",
462 len(inputRatios), len(inputFluxes)
if inputFluxes
else 0)
464 if inputFluxes
is None:
465 inputFluxes = [
None for exp
in inputRatios]
467 combinedRatios = defaultdict(
lambda: defaultdict(list))
468 combinedFluxes = defaultdict(
lambda: defaultdict(list))
469 for ratioDict, fluxDict
in zip(inputRatios, inputFluxes):
470 for targetChip
in ratioDict:
471 if calibChip
and targetChip != calibChip:
472 raise RuntimeError(
"Received multiple target chips!")
474 sourceChip = targetChip
475 if sourceChip
in ratioDict[targetChip]:
476 ratios = ratioDict[targetChip][sourceChip]
478 for targetAmp
in ratios:
479 for sourceAmp
in ratios[targetAmp]:
480 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
482 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
487 for targetAmp
in combinedRatios:
488 for sourceAmp
in combinedRatios[targetAmp]:
489 self.log.info(
"Read %d pixels for %s -> %s",
490 len(combinedRatios[targetAmp][sourceAmp]),
491 targetAmp, sourceAmp)
492 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
493 self.
debugRatios(
'reduce', combinedRatios, targetAmp, sourceAmp)
495 if self.config.fluxOrder == 0:
496 self.log.info(
"Fitting crosstalk coefficients.")
498 self.config.rejIter, self.config.rejSigma)
500 raise NotImplementedError(
"Non-linear crosstalk terms are not yet supported.")
502 self.log.info(
"Number of valid coefficients: %d", np.sum(calib.coeffValid))
504 if self.config.doFiltering:
507 self.log.info(
"Filtering measured crosstalk to remove invalid solutions.")
511 calib.hasCrosstalk =
True
513 calib._detectorName = calibChip
516 if chip.getName() == calibChip:
517 calib._detectorSerial = chip.getSerial()
518 calib._instrument = instrument
519 calib.updateMetadata()
522 provenance = IsrProvenance(calibType=
"CROSSTALK")
523 provenance._detectorName = calibChip
525 provenance.fromDataIds(inputDims)
526 provenance._instrument = instrument
527 provenance.updateMetadata()
529 return pipeBase.Struct(
530 outputCrosstalk=calib,
531 outputProvenance=provenance,
535 """Measure crosstalk coefficients from the ratios.
537 Given a list of ratios for each target/source amp combination,
538 we measure a sigma clipped mean and error.
540 The coefficient errors returned are the standard deviation of
541 the final set of clipped input ratios.
545 ratios : `dict` of `dict` of `numpy.ndarray`
546 Catalog of arrays of ratios.
548 Number of rejection iterations.
550 Rejection threshold (sigma).
554 calib : `lsst.ip.isr.CrosstalkCalib`
555 The output crosstalk calibration.
559 The lsstDebug.Info() method can be rewritten for __name__ =
560 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
562 debug.display['measure'] : `bool`
563 Display the CDF of the combined ratio measurements for
564 a pair of source/target amplifiers from the final set of
565 clipped input ratios.
567 calib = CrosstalkCalib(nAmp=len(ratios))
570 ordering = list(ratios.keys())
571 for ii, jj
in itertools.product(range(calib.nAmp), range(calib.nAmp)):
575 values = np.array(ratios[ordering[ii]][ordering[jj]])
576 values = values[np.abs(values) < 1.0]
578 calib.coeffNum[ii][jj] = len(values)
581 self.log.warn(
"No values for matrix element %d,%d" % (ii, jj))
582 calib.coeffs[ii][jj] = np.nan
583 calib.coeffErr[ii][jj] = np.nan
584 calib.coeffValid[ii][jj] =
False
587 for rej
in range(rejIter):
588 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
589 sigma = 0.741*(hi - lo)
590 good = np.abs(values - med) < rejSigma*sigma
591 if good.sum() == len(good):
593 values = values[good]
595 calib.coeffs[ii][jj] = np.mean(values)
596 if calib.coeffNum[ii][jj] == 1:
597 calib.coeffErr[ii][jj] = np.nan
600 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
601 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) >
602 calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
604 if calib.coeffNum[ii][jj] > 1:
605 self.
debugRatios(
'measure', ratios, ordering[ii], ordering[jj],
606 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
612 """Correct measured sigma to account for clipping.
614 If we clip our input data and then measure sigma, then the
615 measured sigma is smaller than the true value because real
616 points beyond the clip threshold have been removed. This is a
617 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the
618 default parameters for measure crosstalk use nSigClip=2.0.
619 This causes the measured sigma to be about 15% smaller than
620 real. This formula corrects the issue, for the symmetric case
621 (upper clip threshold equal to lower clip threshold).
626 Number of sigma the measurement was clipped by.
630 scaleFactor : `float`
631 Scale factor to increase the measured sigma by.
634 varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip))
635 return 1.0 / np.sqrt(varFactor)
639 """Apply valid constraints to the measured values.
641 Any measured coefficient that is determined to be invalid is
642 set to zero, and has the error set to nan. The validation is
643 determined by checking that the measured coefficient is larger
644 than the calculated standard error of the mean.
648 inCalib : `lsst.ip.isr.CrosstalkCalib`
649 Input calibration to filter.
653 outCalib : `lsst.ip.isr.CrosstalkCalib`
654 Filtered calibration.
656 outCalib = CrosstalkCalib()
657 outCalib.numAmps = inCalib.numAmps
659 outCalib.coeffs = inCalib.coeffs
660 outCalib.coeffs[~inCalib.coeffValid] = 0.0
662 outCalib.coeffErr = inCalib.coeffErr
663 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
665 outCalib.coeffNum = inCalib.coeffNum
666 outCalib.coeffValid = inCalib.coeffValid
670 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
671 """Utility function to examine the final CT ratio set.
676 State of processing to view.
677 ratios : `dict` of `dict` of `np.ndarray`
678 Array of measured CT ratios, indexed by source/victim
681 Index of the source amplifier.
683 Index of the target amplifier.
684 coeff : `float`, optional
685 Coefficient calculated to plot along with the simple mean.
686 valid : `bool`, optional
687 Validity to be added to the plot title.
689 frame = getDebugFrame(self._display, stepname)
691 if i == j
or ratios
is None or len(ratios) < 1:
694 ratioList = ratios[i][j]
695 if ratioList
is None or len(ratioList) < 1:
698 mean = np.mean(ratioList)
699 std = np.std(ratioList)
700 import matplotlib.pyplot
as plt
701 figure = plt.figure(1)
703 plt.hist(x=ratioList, bins=len(ratioList),
704 cumulative=
True, color=
'b', density=
True, histtype=
'step')
705 plt.xlabel(
"Measured pixel ratio")
706 plt.ylabel(f
"CDF: n={len(ratioList)}")
707 plt.xlim(np.percentile(ratioList, [1.0, 99]))
708 plt.axvline(x=mean, color=
"k")
709 plt.axvline(x=coeff, color=
'g')
710 plt.axvline(x=(std / np.sqrt(len(ratioList))), color=
'r')
711 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color=
'r')
712 plt.title(f
"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
715 prompt =
"Press Enter to continue: "
717 ans = input(prompt).lower()
718 if ans
in (
"",
"c",):
720 elif ans
in (
"pdb",
"p",):
727 extract = ConfigurableField(
728 target=CrosstalkExtractTask,
729 doc=
"Task to measure pixel ratios.",
731 solver = ConfigurableField(
732 target=CrosstalkSolveTask,
733 doc=
"Task to convert ratio lists to crosstalk coefficients.",
738 """Measure intra-detector crosstalk.
742 lsst.ip.isr.crosstalk.CrosstalkCalib
743 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
744 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
748 The crosstalk this method measures assumes that when a bright
749 pixel is found in one detector amplifier, all other detector
750 amplifiers may see a signal change in the same pixel location
751 (relative to the readout amplifier) as these other pixels are read
752 out at the same time.
754 After processing each input exposure through a limited set of ISR
755 stages, bright unmasked pixels above the threshold are identified.
756 The potential CT signal is found by taking the ratio of the
757 appropriate background-subtracted pixel value on the other
758 amplifiers to the input value on the source amplifier. If the
759 source amplifier has a large number of bright pixels as well, the
760 background level may be elevated, leading to poor ratio
763 The set of ratios found between each pair of amplifiers across all
764 input exposures is then gathered to produce the final CT
765 coefficients. The sigma-clipped mean and sigma are returned from
766 these sets of ratios, with the coefficient to supply to the ISR
767 CrosstalkTask() being the multiplicative inverse of these values.
769 This Task simply calls the pipetask versions of the measure
772 ConfigClass = MeasureCrosstalkConfig
773 _DefaultName =
"measureCrosstalk"
776 RunnerClass = DataRefListRunner
780 self.makeSubtask(
"extract")
781 self.makeSubtask(
"solver")
784 """Run extract task on each of inputs in the dataRef list, then pass
785 that to the solver task.
789 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
790 Data references for exposures for detectors to process.
794 results : `lsst.pipe.base.Struct`
795 The results struct containing:
797 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
798 Final crosstalk calibration.
799 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
800 Provenance data for the new calibration.
805 Raised if multiple target detectors are supplied.
807 dataRef = dataRefList[0]
808 camera = dataRef.get(
"camera")
812 for dataRef
in dataRefList:
813 exposure = dataRef.get(
"postISRCCD")
815 if exposure.getDetector().getName() != activeChip:
816 raise RuntimeError(
"Too many input detectors supplied!")
818 activeChip = exposure.getDetector().getName()
820 self.extract.debugView(
"extract", exposure)
821 result = self.extract.run(exposure)
822 ratios.append(result.outputRatios)
824 finalResults = self.solver.run(ratios, camera=camera)
825 dataRef.put(finalResults.outputCrosstalk,
"crosstalk")