23 from scipy.stats
import norm
25 from collections
import defaultdict
30 from lsstDebug
import getDebugFrame
34 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
35 from lsst.pipe.tasks.getRepositoryData
import DataRefListRunner
37 __all__ = [
"CrosstalkExtractConfig",
"CrosstalkExtractTask",
38 "CrosstalkSolveTask",
"CrosstalkSolveConfig",
39 "MeasureCrosstalkConfig",
"MeasureCrosstalkTask"]
43 dimensions=(
"instrument",
"exposure",
"detector")):
45 name=
"crosstalkInputs",
46 doc=
"Input post-ISR processed exposure to measure crosstalk from.",
47 storageClass=
"Exposure",
48 dimensions=(
"instrument",
"exposure",
"detector"),
53 name=
"crosstalkSource",
54 doc=
"Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
55 storageClass=
"Exposure",
56 dimensions=(
"instrument",
"exposure",
"detector"),
62 outputRatios = cT.Output(
63 name=
"crosstalkRatios",
64 doc=
"Extracted crosstalk pixel ratios.",
65 storageClass=
"StructuredDataDict",
66 dimensions=(
"instrument",
"exposure",
"detector"),
68 outputFluxes = cT.Output(
69 name=
"crosstalkFluxes",
70 doc=
"Source pixel fluxes used in ratios.",
71 storageClass=
"StructuredDataDict",
72 dimensions=(
"instrument",
"exposure",
"detector"),
79 self.inputs.discard(
"sourceExp")
83 pipelineConnections=CrosstalkExtractConnections):
84 """Configuration for the measurement of pixel ratios.
86 doMeasureInterchip = Field(
89 doc=
"Measure inter-chip crosstalk as well?",
94 doc=
"Minimum level of source pixels for which to measure crosstalk."
96 ignoreSaturatedPixels = Field(
99 doc=
"Should saturated pixels be ignored?"
103 default=[
"BAD",
"INTRP"],
104 doc=
"Mask planes to ignore when identifying source pixels."
109 doc=
"Is the input exposure trimmed?"
126 pipeBase.CmdLineTask):
127 """Task to measure pixel ratios to find crosstalk.
129 ConfigClass = CrosstalkExtractConfig
130 _DefaultName =
'cpCrosstalkExtract'
132 def run(self, inputExp, sourceExps=[]):
133 """Measure pixel ratios between amplifiers in inputExp.
135 Extract crosstalk ratios between different amplifiers.
137 For pixels above ``config.threshold``, we calculate the ratio
138 between each background-subtracted target amp and the source
139 amp. We return a list of ratios for each pixel for each
140 target/source combination, as nested dictionary containing the
145 inputExp : `lsst.afw.image.Exposure`
146 Input exposure to measure pixel ratios on.
147 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
148 List of chips to use as sources to measure inter-chip
153 results : `lsst.pipe.base.Struct`
154 The results struct containing:
156 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
157 A catalog of ratio lists. The dictionaries are
159 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
160 contains the ratio list for that combination.
161 ``outputFluxes`` : `dict` [`dict` [`list`]]
162 A catalog of flux lists. The dictionaries are
164 outputFluxes[sourceChip][sourceAmp]
165 contains the flux list used in the outputRatios.
169 The lsstDebug.Info() method can be rewritten for __name__ =
170 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
172 debug.display['extract'] : `bool`
173 Display the exposure under consideration, with the pixels used
174 for crosstalk measurement indicated by the DETECTED mask plane.
175 debug.display['pixels'] : `bool`
176 Display a plot of the ratio calculated for each pixel used in this
177 exposure, split by amplifier pairs. The median value is listed
180 outputRatios = defaultdict(
lambda: defaultdict(dict))
181 outputFluxes = defaultdict(
lambda: defaultdict(dict))
183 threshold = self.config.threshold
184 badPixels = list(self.config.badMask)
186 targetDetector = inputExp.getDetector()
187 targetChip = targetDetector.getName()
190 sourceExtractExps = [inputExp]
191 sourceExtractExps.extend(sourceExps)
193 self.log.info(
"Measuring full detector background for target: %s", targetChip)
194 targetIm = inputExp.getMaskedImage()
196 detected = targetIm.getMask().getPlaneBitMask(
"DETECTED")
197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + [
"DETECTED"])
201 for sourceExp
in sourceExtractExps:
202 sourceDetector = sourceExp.getDetector()
203 sourceChip = sourceDetector.getName()
204 sourceIm = sourceExp.getMaskedImage()
205 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
206 self.log.info(
"Measuring crosstalk from source: %s", sourceChip)
208 if sourceExp != inputExp:
210 detected = sourceIm.getMask().getPlaneBitMask(
"DETECTED")
213 ratioDict = defaultdict(
lambda: defaultdict(list))
216 for sourceAmp
in sourceDetector:
217 sourceAmpName = sourceAmp.getName()
218 sourceAmpImage = sourceIm[sourceAmp.getBBox()]
219 sourceMask = sourceAmpImage.mask.array
220 select = ((sourceMask & detected > 0) &
221 (sourceMask & bad == 0) &
222 np.isfinite(sourceAmpImage.image.array))
223 count = np.sum(select)
224 self.log.debug(
" Source amplifier: %s", sourceAmpName)
226 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select]
228 for targetAmp
in targetDetector:
230 targetAmpName = targetAmp.getName()
231 if sourceAmpName == targetAmpName
and sourceChip == targetChip:
232 ratioDict[sourceAmpName][targetAmpName] = []
234 self.log.debug(
" Target amplifier: %s", targetAmpName)
236 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
237 targetAmp, sourceAmp,
238 isTrimmed=self.config.isTrimmed)
239 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
240 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
241 extractedCount += count
244 sourceAmpImage.image.array[select],
245 targetAmpImage.array[select] - bg,
246 sourceAmpName, targetAmpName)
248 self.log.info(
"Extracted %d pixels from %s -> %s (targetBG: %f)",
249 extractedCount, sourceChip, targetChip, bg)
250 outputRatios[targetChip][sourceChip] = ratioDict
252 return pipeBase.Struct(
253 outputRatios=outputRatios,
254 outputFluxes=outputFluxes
258 """Utility function to examine the image being processed.
263 State of processing to view.
264 exposure : `lsst.afw.image.Exposure`
267 frame = getDebugFrame(self._display, stepname)
269 display = getDisplay(frame)
270 display.scale(
'asinh',
'zscale')
271 display.mtv(exposure)
273 prompt =
"Press Enter to continue: "
275 ans = input(prompt).lower()
276 if ans
in (
"",
"c",):
279 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
280 """Utility function to examine the CT ratio pixel values.
285 State of processing to view.
286 pixelsIn : `np.ndarray`
287 Pixel values from the potential crosstalk source.
288 pixelsOut : `np.ndarray`
289 Pixel values from the potential crosstalk target.
291 Source amplifier name
293 Target amplifier name
295 frame = getDebugFrame(self._display, stepname)
297 import matplotlib.pyplot
as plt
298 figure = plt.figure(1)
301 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
302 axes.plt(pixelsIn, pixelsOut / pixelsIn,
'k+')
303 plt.xlabel(
"Source amplifier pixel value")
304 plt.ylabel(
"Measured pixel ratio")
305 plt.title(f
"(Source {sourceName} -> Target {targetName}) median ratio: "
306 f
"{(np.median(pixelsOut / pixelsIn))}")
309 prompt =
"Press Enter to continue: "
311 ans = input(prompt).lower()
312 if ans
in (
"",
"c",):
318 dimensions=(
"instrument",
"detector")):
319 inputRatios = cT.Input(
320 name=
"crosstalkRatios",
321 doc=
"Ratios measured for an input exposure.",
322 storageClass=
"StructuredDataDict",
323 dimensions=(
"instrument",
"exposure",
"detector"),
326 inputFluxes = cT.Input(
327 name=
"crosstalkFluxes",
328 doc=
"Fluxes of CT source pixels, for nonlinear fits.",
329 storageClass=
"StructuredDataDict",
330 dimensions=(
"instrument",
"exposure",
"detector"),
333 camera = cT.PrerequisiteInput(
335 doc=
"Camera the input data comes from.",
336 storageClass=
"Camera",
337 dimensions=(
"instrument",
"calibration_label"),
340 outputCrosstalk = cT.Output(
341 name=
"crosstalkProposal",
342 doc=
"Output proposed crosstalk calibration.",
343 storageClass=
"CrosstalkCalib",
344 dimensions=(
"instrument",
"detector"),
351 if config.fluxOrder == 0:
352 self.Inputs.discard(
"inputFluxes")
356 pipelineConnections=CrosstalkSolveConnections):
357 """Configuration for the solving of crosstalk from pixel ratios.
362 doc=
"Number of rejection iterations for final coefficient calculation.",
367 doc=
"Rejection threshold (sigma) for final coefficient calculation.",
372 doc=
"Polynomial order in source flux to fit crosstalk.",
377 doc=
"Filter generated crosstalk to remove marginal measurements.",
382 pipeBase.CmdLineTask):
383 """Task to solve crosstalk from pixel ratios.
385 ConfigClass = CrosstalkSolveConfig
386 _DefaultName =
'cpCrosstalkSolve'
389 """Ensure that the input and output dimensions are passed along.
393 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
394 Butler to operate on.
395 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
396 Input data refs to load.
397 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
398 Output data refs to persist.
400 inputs = butlerQC.get(inputRefs)
403 inputs[
'inputDims'] = [exp.dataId.byName()
for exp
in inputRefs.inputRatios]
404 inputs[
'outputDims'] = [exp.dataId.byName()
for exp
in outputRefs.outputCrosstalk]
406 outputs = self.
run(**inputs)
407 butlerQC.put(outputs, outputRefs)
409 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
410 """Combine ratios to produce crosstalk coefficients.
414 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
415 A list of nested dictionaries of ratios indexed by target
416 and source chip, then by target and source amplifier.
417 inputFluxes : `list` [`dict` [`dict` [`list`]]]
418 A list of nested dictionaries of source pixel fluxes, indexed
419 by source chip and amplifier.
420 camera : `lsst.afw.cameraGeom.Camera`
422 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
423 DataIds to use to construct provenance.
424 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
425 DataIds to use to populate the output calibration.
429 results : `lsst.pipe.base.Struct`
430 The results struct containing:
432 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
433 Final crosstalk calibration.
434 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
435 Provenance data for the new calibration.
440 Raised if the input data contains multiple target detectors.
444 The lsstDebug.Info() method can be rewritten for __name__ =
445 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
447 debug.display['reduce'] : `bool`
448 Display a histogram of the combined ratio measurements for
449 a pair of source/target amplifiers from all input
454 calibChip = outputDims[
'detector']
459 self.log.info(
"Combining measurements from %d ratios and %d fluxes",
460 len(inputRatios), len(inputFluxes)
if inputFluxes
else 0)
462 if inputFluxes
is None:
463 inputFluxes = [
None for exp
in inputRatios]
465 combinedRatios = defaultdict(
lambda: defaultdict(list))
466 combinedFluxes = defaultdict(
lambda: defaultdict(list))
467 for ratioDict, fluxDict
in zip(inputRatios, inputFluxes):
468 for targetChip
in ratioDict:
469 if calibChip
and targetChip != calibChip:
470 raise RuntimeError(
"Received multiple target chips!")
472 sourceChip = targetChip
473 if sourceChip
in ratioDict[targetChip]:
474 ratios = ratioDict[targetChip][sourceChip]
476 for targetAmp
in ratios:
477 for sourceAmp
in ratios[targetAmp]:
478 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
480 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
485 for targetAmp
in combinedRatios:
486 for sourceAmp
in combinedRatios[targetAmp]:
487 self.log.info(
"Read %d pixels for %s -> %s",
488 len(combinedRatios[targetAmp][sourceAmp]),
489 targetAmp, sourceAmp)
490 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
491 self.
debugRatios(
'reduce', combinedRatios, targetAmp, sourceAmp)
493 if self.config.fluxOrder == 0:
494 self.log.info(
"Fitting crosstalk coefficients.")
496 self.config.rejIter, self.config.rejSigma)
498 raise NotImplementedError(
"Non-linear crosstalk terms are not yet supported.")
500 self.log.info(
"Number of valid coefficients: %d", np.sum(calib.coeffValid))
502 if self.config.doFiltering:
505 self.log.info(
"Filtering measured crosstalk to remove invalid solutions.")
509 calib.hasCrosstalk =
True
511 calib._detectorName = calibChip
514 if chip.getName() == calibChip:
515 calib._detectorSerial = chip.getSerial()
516 calib.updateMetadata()
519 provenance = IsrProvenance(calibType=
"CROSSTALK")
520 provenance._detectorName = calibChip
522 provenance.fromDataIds(inputDims)
523 provenance._instrument = outputDims[
'instrument']
524 provenance.updateMetadata()
526 return pipeBase.Struct(
527 outputCrosstalk=calib,
528 outputProvenance=provenance,
532 """Measure crosstalk coefficients from the ratios.
534 Given a list of ratios for each target/source amp combination,
535 we measure a sigma clipped mean and error.
537 The coefficient errors returned are the standard deviation of
538 the final set of clipped input ratios.
542 ratios : `dict` of `dict` of `numpy.ndarray`
543 Catalog of arrays of ratios.
545 Number of rejection iterations.
547 Rejection threshold (sigma).
551 calib : `lsst.ip.isr.CrosstalkCalib`
552 The output crosstalk calibration.
556 The lsstDebug.Info() method can be rewritten for __name__ =
557 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
559 debug.display['measure'] : `bool`
560 Display the CDF of the combined ratio measurements for
561 a pair of source/target amplifiers from the final set of
562 clipped input ratios.
564 calib = CrosstalkCalib(nAmp=len(ratios))
567 ordering = list(ratios.keys())
568 for ii, jj
in itertools.product(range(calib.nAmp), range(calib.nAmp)):
572 values = np.array(ratios[ordering[ii]][ordering[jj]])
573 values = values[np.abs(values) < 1.0]
575 calib.coeffNum[ii][jj] = len(values)
578 self.log.warn(
"No values for matrix element %d,%d" % (ii, jj))
579 calib.coeffs[ii][jj] = np.nan
580 calib.coeffErr[ii][jj] = np.nan
581 calib.coeffValid[ii][jj] =
False
584 for rej
in range(rejIter):
585 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
586 sigma = 0.741*(hi - lo)
587 good = np.abs(values - med) < rejSigma*sigma
588 if good.sum() == len(good):
590 values = values[good]
592 calib.coeffs[ii][jj] = np.mean(values)
593 if calib.coeffNum[ii][jj] == 1:
594 calib.coeffErr[ii][jj] = np.nan
597 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
598 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) >
599 calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
601 if calib.coeffNum[ii][jj] > 1:
602 self.
debugRatios(
'measure', ratios, ordering[ii], ordering[jj],
603 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
609 """Correct measured sigma to account for clipping.
611 If we clip our input data and then measure sigma, then the
612 measured sigma is smaller than the true value because real
613 points beyond the clip threshold have been removed. This is a
614 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the
615 default parameters for measure crosstalk use nSigClip=2.0.
616 This causes the measured sigma to be about 15% smaller than
617 real. This formula corrects the issue, for the symmetric case
618 (upper clip threshold equal to lower clip threshold).
623 Number of sigma the measurement was clipped by.
627 scaleFactor : `float`
628 Scale factor to increase the measured sigma by.
631 varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip))
632 return 1.0 / np.sqrt(varFactor)
636 """Apply valid constraints to the measured values.
638 Any measured coefficient that is determined to be invalid is
639 set to zero, and has the error set to nan. The validation is
640 determined by checking that the measured coefficient is larger
641 than the calculated standard error of the mean.
645 inCalib : `lsst.ip.isr.CrosstalkCalib`
646 Input calibration to filter.
650 outCalib : `lsst.ip.isr.CrosstalkCalib`
651 Filtered calibration.
653 outCalib = CrosstalkCalib()
654 outCalib.numAmps = inCalib.numAmps
656 outCalib.coeffs = inCalib.coeffs
657 outCalib.coeffs[~inCalib.coeffValid] = 0.0
659 outCalib.coeffErr = inCalib.coeffErr
660 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
662 outCalib.coeffNum = inCalib.coeffNum
663 outCalib.coeffValid = inCalib.coeffValid
667 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
668 """Utility function to examine the final CT ratio set.
673 State of processing to view.
674 ratios : `dict` of `dict` of `np.ndarray`
675 Array of measured CT ratios, indexed by source/victim
678 Index of the source amplifier.
680 Index of the target amplifier.
681 coeff : `float`, optional
682 Coefficient calculated to plot along with the simple mean.
683 valid : `bool`, optional
684 Validity to be added to the plot title.
686 frame = getDebugFrame(self._display, stepname)
688 if i == j
or ratios
is None or len(ratios) < 1:
691 ratioList = ratios[i][j]
692 if ratioList
is None or len(ratioList) < 1:
695 mean = np.mean(ratioList)
696 std = np.std(ratioList)
697 import matplotlib.pyplot
as plt
698 figure = plt.figure(1)
700 plt.hist(x=ratioList, bins=len(ratioList),
701 cumulative=
True, color=
'b', density=
True, histtype=
'step')
702 plt.xlabel(
"Measured pixel ratio")
703 plt.ylabel(f
"CDF: n={len(ratioList)}")
704 plt.xlim(np.percentile(ratioList, [1.0, 99]))
705 plt.axvline(x=mean, color=
"k")
706 plt.axvline(x=coeff, color=
'g')
707 plt.axvline(x=(std / np.sqrt(len(ratioList))), color=
'r')
708 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color=
'r')
709 plt.title(f
"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
712 prompt =
"Press Enter to continue: "
714 ans = input(prompt).lower()
715 if ans
in (
"",
"c",):
717 elif ans
in (
"pdb",
"p",):
724 extract = ConfigurableField(
725 target=CrosstalkExtractTask,
726 doc=
"Task to measure pixel ratios.",
728 solver = ConfigurableField(
729 target=CrosstalkSolveTask,
730 doc=
"Task to convert ratio lists to crosstalk coefficients.",
735 """Measure intra-detector crosstalk.
739 lsst.ip.isr.crosstalk.CrosstalkCalib
740 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
741 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
745 The crosstalk this method measures assumes that when a bright
746 pixel is found in one detector amplifier, all other detector
747 amplifiers may see a signal change in the same pixel location
748 (relative to the readout amplifier) as these other pixels are read
749 out at the same time.
751 After processing each input exposure through a limited set of ISR
752 stages, bright unmasked pixels above the threshold are identified.
753 The potential CT signal is found by taking the ratio of the
754 appropriate background-subtracted pixel value on the other
755 amplifiers to the input value on the source amplifier. If the
756 source amplifier has a large number of bright pixels as well, the
757 background level may be elevated, leading to poor ratio
760 The set of ratios found between each pair of amplifiers across all
761 input exposures is then gathered to produce the final CT
762 coefficients. The sigma-clipped mean and sigma are returned from
763 these sets of ratios, with the coefficient to supply to the ISR
764 CrosstalkTask() being the multiplicative inverse of these values.
766 This Task simply calls the pipetask versions of the measure
769 ConfigClass = MeasureCrosstalkConfig
770 _DefaultName =
"measureCrosstalk"
773 RunnerClass = DataRefListRunner
777 self.makeSubtask(
"extract")
778 self.makeSubtask(
"solver")
781 """Run extract task on each of inputs in the dataRef list, then pass
782 that to the solver task.
786 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
787 Data references for exposures for detectors to process.
791 results : `lsst.pipe.base.Struct`
792 The results struct containing:
794 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
795 Final crosstalk calibration.
796 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
797 Provenance data for the new calibration.
802 Raised if multiple target detectors are supplied.
804 dataRef = dataRefList[0]
805 camera = dataRef.get(
"camera")
809 for dataRef
in dataRefList:
810 exposure = dataRef.get(
"postISRCCD")
812 if exposure.getDetector().getName() != activeChip:
813 raise RuntimeError(
"Too many input detectors supplied!")
815 activeChip = exposure.getDetector().getName()
817 self.extract.debugView(
"extract", exposure)
818 result = self.extract.run(exposure)
819 ratios.append(result.outputRatios)
821 finalResults = self.solver.run(ratios, camera=camera)
822 dataRef.put(finalResults.outputCrosstalk,
"crosstalk")