24 from collections
import defaultdict
29 from lsstDebug
import getDebugFrame
33 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
34 from lsst.pipe.tasks.getRepositoryData
import DataRefListRunner
37 from ._lookupStaticCalibration
import lookupStaticCalibration
39 __all__ = [
"CrosstalkExtractConfig",
"CrosstalkExtractTask",
40 "CrosstalkSolveTask",
"CrosstalkSolveConfig",
41 "MeasureCrosstalkConfig",
"MeasureCrosstalkTask"]
45 dimensions=(
"instrument",
"exposure",
"detector")):
47 name=
"crosstalkInputs",
48 doc=
"Input post-ISR processed exposure to measure crosstalk from.",
49 storageClass=
"Exposure",
50 dimensions=(
"instrument",
"exposure",
"detector"),
55 name=
"crosstalkSource",
56 doc=
"Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
57 storageClass=
"Exposure",
58 dimensions=(
"instrument",
"exposure",
"detector"),
64 outputRatios = cT.Output(
65 name=
"crosstalkRatios",
66 doc=
"Extracted crosstalk pixel ratios.",
67 storageClass=
"StructuredDataDict",
68 dimensions=(
"instrument",
"exposure",
"detector"),
70 outputFluxes = cT.Output(
71 name=
"crosstalkFluxes",
72 doc=
"Source pixel fluxes used in ratios.",
73 storageClass=
"StructuredDataDict",
74 dimensions=(
"instrument",
"exposure",
"detector"),
81 self.inputs.discard(
"sourceExp")
85 pipelineConnections=CrosstalkExtractConnections):
86 """Configuration for the measurement of pixel ratios.
88 doMeasureInterchip = Field(
91 doc=
"Measure inter-chip crosstalk as well?",
96 doc=
"Minimum level of source pixels for which to measure crosstalk."
98 ignoreSaturatedPixels = Field(
101 doc=
"Should saturated pixels be ignored?"
105 default=[
"BAD",
"INTRP"],
106 doc=
"Mask planes to ignore when identifying source pixels."
111 doc=
"Is the input exposure trimmed?"
120 if 'SAT' not in self.
badMaskbadMask:
121 self.
badMaskbadMask.append(
'SAT')
123 if 'SAT' in self.
badMaskbadMask:
124 self.
badMaskbadMask = [mask
for mask
in self.
badMaskbadMask
if mask !=
'SAT']
128 pipeBase.CmdLineTask):
129 """Task to measure pixel ratios to find crosstalk.
131 ConfigClass = CrosstalkExtractConfig
132 _DefaultName =
'cpCrosstalkExtract'
134 def run(self, inputExp, sourceExps=[]):
135 """Measure pixel ratios between amplifiers in inputExp.
137 Extract crosstalk ratios between different amplifiers.
139 For pixels above ``config.threshold``, we calculate the ratio
140 between each background-subtracted target amp and the source
141 amp. We return a list of ratios for each pixel for each
142 target/source combination, as nested dictionary containing the
147 inputExp : `lsst.afw.image.Exposure`
148 Input exposure to measure pixel ratios on.
149 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
150 List of chips to use as sources to measure inter-chip
155 results : `lsst.pipe.base.Struct`
156 The results struct containing:
158 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
159 A catalog of ratio lists. The dictionaries are
161 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
162 contains the ratio list for that combination.
163 ``outputFluxes`` : `dict` [`dict` [`list`]]
164 A catalog of flux lists. The dictionaries are
166 outputFluxes[sourceChip][sourceAmp]
167 contains the flux list used in the outputRatios.
171 The lsstDebug.Info() method can be rewritten for __name__ =
172 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
174 debug.display['extract'] : `bool`
175 Display the exposure under consideration, with the pixels used
176 for crosstalk measurement indicated by the DETECTED mask plane.
177 debug.display['pixels'] : `bool`
178 Display a plot of the ratio calculated for each pixel used in this
179 exposure, split by amplifier pairs. The median value is listed
182 outputRatios = defaultdict(
lambda: defaultdict(dict))
183 outputFluxes = defaultdict(
lambda: defaultdict(dict))
185 threshold = self.config.threshold
186 badPixels = list(self.config.badMask)
188 targetDetector = inputExp.getDetector()
189 targetChip = targetDetector.getName()
192 sourceExtractExps = [inputExp]
193 sourceExtractExps.extend(sourceExps)
195 self.log.info(
"Measuring full detector background for target: %s", targetChip)
196 targetIm = inputExp.getMaskedImage()
198 detected = targetIm.getMask().getPlaneBitMask(
"DETECTED")
199 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + [
"DETECTED"])
201 self.
debugViewdebugView(
'extract', inputExp)
203 for sourceExp
in sourceExtractExps:
204 sourceDetector = sourceExp.getDetector()
205 sourceChip = sourceDetector.getName()
206 sourceIm = sourceExp.getMaskedImage()
207 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
208 self.log.info(
"Measuring crosstalk from source: %s", sourceChip)
210 if sourceExp != inputExp:
212 detected = sourceIm.getMask().getPlaneBitMask(
"DETECTED")
215 ratioDict = defaultdict(
lambda: defaultdict(list))
218 for sourceAmp
in sourceDetector:
219 sourceAmpName = sourceAmp.getName()
220 sourceAmpBBox = sourceAmp.getBBox()
if self.config.isTrimmed
else sourceAmp.getRawDataBBox()
221 sourceAmpImage = sourceIm[sourceAmpBBox]
222 sourceMask = sourceAmpImage.mask.array
223 select = ((sourceMask & detected > 0)
224 & (sourceMask & bad == 0)
225 & np.isfinite(sourceAmpImage.image.array))
226 count = np.sum(select)
227 self.log.debug(
" Source amplifier: %s", sourceAmpName)
229 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
231 for targetAmp
in targetDetector:
233 targetAmpName = targetAmp.getName()
234 if sourceAmpName == targetAmpName
and sourceChip == targetChip:
235 ratioDict[sourceAmpName][targetAmpName] = []
237 self.log.debug(
" Target amplifier: %s", targetAmpName)
239 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
240 targetAmp, sourceAmp,
241 isTrimmed=self.config.isTrimmed)
242 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
243 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
244 extractedCount += count
247 sourceAmpImage.image.array[select],
248 targetAmpImage.array[select] - bg,
249 sourceAmpName, targetAmpName)
251 self.log.info(
"Extracted %d pixels from %s -> %s (targetBG: %f)",
252 extractedCount, sourceChip, targetChip, bg)
253 outputRatios[targetChip][sourceChip] = ratioDict
255 return pipeBase.Struct(
261 """Utility function to examine the image being processed.
266 State of processing to view.
267 exposure : `lsst.afw.image.Exposure`
270 frame = getDebugFrame(self._display, stepname)
272 display = getDisplay(frame)
273 display.scale(
'asinh',
'zscale')
274 display.mtv(exposure)
276 prompt =
"Press Enter to continue: "
278 ans = input(prompt).lower()
279 if ans
in (
"",
"c",):
282 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
283 """Utility function to examine the CT ratio pixel values.
288 State of processing to view.
289 pixelsIn : `np.ndarray`
290 Pixel values from the potential crosstalk source.
291 pixelsOut : `np.ndarray`
292 Pixel values from the potential crosstalk target.
294 Source amplifier name
296 Target amplifier name
298 frame = getDebugFrame(self._display, stepname)
300 import matplotlib.pyplot
as plt
301 figure = plt.figure(1)
304 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
305 axes.plot(pixelsIn, pixelsOut / pixelsIn,
'k+')
306 plt.xlabel(
"Source amplifier pixel value")
307 plt.ylabel(
"Measured pixel ratio")
308 plt.title(f
"(Source {sourceName} -> Target {targetName}) median ratio: "
309 f
"{(np.median(pixelsOut / pixelsIn))}")
312 prompt =
"Press Enter to continue: "
314 ans = input(prompt).lower()
315 if ans
in (
"",
"c",):
321 dimensions=(
"instrument",
"detector")):
322 inputRatios = cT.Input(
323 name=
"crosstalkRatios",
324 doc=
"Ratios measured for an input exposure.",
325 storageClass=
"StructuredDataDict",
326 dimensions=(
"instrument",
"exposure",
"detector"),
329 inputFluxes = cT.Input(
330 name=
"crosstalkFluxes",
331 doc=
"Fluxes of CT source pixels, for nonlinear fits.",
332 storageClass=
"StructuredDataDict",
333 dimensions=(
"instrument",
"exposure",
"detector"),
336 camera = cT.PrerequisiteInput(
338 doc=
"Camera the input data comes from.",
339 storageClass=
"Camera",
340 dimensions=(
"instrument",),
342 lookupFunction=lookupStaticCalibration,
345 outputCrosstalk = cT.Output(
347 doc=
"Output proposed crosstalk calibration.",
348 storageClass=
"CrosstalkCalib",
349 dimensions=(
"instrument",
"detector"),
357 if config.fluxOrder == 0:
358 self.inputs.discard(
"inputFluxes")
362 pipelineConnections=CrosstalkSolveConnections):
363 """Configuration for the solving of crosstalk from pixel ratios.
368 doc=
"Number of rejection iterations for final coefficient calculation.",
373 doc=
"Rejection threshold (sigma) for final coefficient calculation.",
378 doc=
"Polynomial order in source flux to fit crosstalk.",
383 doc=
"Filter generated crosstalk to remove marginal measurements.",
388 pipeBase.CmdLineTask):
389 """Task to solve crosstalk from pixel ratios.
391 ConfigClass = CrosstalkSolveConfig
392 _DefaultName =
'cpCrosstalkSolve'
395 """Ensure that the input and output dimensions are passed along.
399 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
400 Butler to operate on.
401 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
402 Input data refs to load.
403 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
404 Output data refs to persist.
406 inputs = butlerQC.get(inputRefs)
409 inputs[
'inputDims'] = [exp.dataId.byName()
for exp
in inputRefs.inputRatios]
410 inputs[
'outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
412 outputs = self.
runrun(**inputs)
413 butlerQC.put(outputs, outputRefs)
415 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
416 """Combine ratios to produce crosstalk coefficients.
420 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
421 A list of nested dictionaries of ratios indexed by target
422 and source chip, then by target and source amplifier.
423 inputFluxes : `list` [`dict` [`dict` [`list`]]]
424 A list of nested dictionaries of source pixel fluxes, indexed
425 by source chip and amplifier.
426 camera : `lsst.afw.cameraGeom.Camera`
428 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
429 DataIds to use to construct provenance.
430 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
431 DataIds to use to populate the output calibration.
435 results : `lsst.pipe.base.Struct`
436 The results struct containing:
438 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
439 Final crosstalk calibration.
440 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
441 Provenance data for the new calibration.
446 Raised if the input data contains multiple target detectors.
450 The lsstDebug.Info() method can be rewritten for __name__ =
451 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
453 debug.display['reduce'] : `bool`
454 Display a histogram of the combined ratio measurements for
455 a pair of source/target amplifiers from all input
460 calibChip = outputDims[
'detector']
461 instrument = outputDims[
'instrument']
467 self.log.info(
"Combining measurements from %d ratios and %d fluxes",
468 len(inputRatios), len(inputFluxes)
if inputFluxes
else 0)
470 if inputFluxes
is None:
471 inputFluxes = [
None for exp
in inputRatios]
473 combinedRatios = defaultdict(
lambda: defaultdict(list))
474 combinedFluxes = defaultdict(
lambda: defaultdict(list))
475 for ratioDict, fluxDict
in zip(inputRatios, inputFluxes):
476 for targetChip
in ratioDict:
477 if calibChip
and targetChip != calibChip:
478 raise RuntimeError(
"Received multiple target chips!")
480 sourceChip = targetChip
481 if sourceChip
in ratioDict[targetChip]:
482 ratios = ratioDict[targetChip][sourceChip]
484 for targetAmp
in ratios:
485 for sourceAmp
in ratios[targetAmp]:
486 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
488 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
493 for targetAmp
in combinedRatios:
494 for sourceAmp
in combinedRatios[targetAmp]:
495 self.log.info(
"Read %d pixels for %s -> %s",
496 len(combinedRatios[targetAmp][sourceAmp]),
497 targetAmp, sourceAmp)
498 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
499 self.
debugRatiosdebugRatios(
'reduce', combinedRatios, targetAmp, sourceAmp)
501 if self.config.fluxOrder == 0:
502 self.log.info(
"Fitting crosstalk coefficients.")
504 self.config.rejIter, self.config.rejSigma)
506 raise NotImplementedError(
"Non-linear crosstalk terms are not yet supported.")
508 self.log.info(
"Number of valid coefficients: %d", np.sum(calib.coeffValid))
510 if self.config.doFiltering:
513 self.log.info(
"Filtering measured crosstalk to remove invalid solutions.")
517 calib.hasCrosstalk =
True
521 calib._detectorId = calibChip
523 calib._detectorName = camera[calibChip].getName()
524 calib._detectorSerial = camera[calibChip].getSerial()
526 calib._instrument = instrument
527 calib.updateMetadata(setCalibId=
True, setDate=
True)
530 provenance = IsrProvenance(calibType=
"CROSSTALK")
531 provenance._detectorName = calibChip
533 provenance.fromDataIds(inputDims)
534 provenance._instrument = instrument
535 provenance.updateMetadata()
537 return pipeBase.Struct(
538 outputCrosstalk=calib,
539 outputProvenance=provenance,
543 """Measure crosstalk coefficients from the ratios.
545 Given a list of ratios for each target/source amp combination,
546 we measure a sigma clipped mean and error.
548 The coefficient errors returned are the standard deviation of
549 the final set of clipped input ratios.
553 ratios : `dict` of `dict` of `numpy.ndarray`
554 Catalog of arrays of ratios.
556 Number of rejection iterations.
558 Rejection threshold (sigma).
562 calib : `lsst.ip.isr.CrosstalkCalib`
563 The output crosstalk calibration.
567 The lsstDebug.Info() method can be rewritten for __name__ =
568 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
570 debug.display['measure'] : `bool`
571 Display the CDF of the combined ratio measurements for
572 a pair of source/target amplifiers from the final set of
573 clipped input ratios.
575 calib = CrosstalkCalib(nAmp=len(ratios))
578 ordering = list(ratios.keys())
579 for ii, jj
in itertools.product(range(calib.nAmp), range(calib.nAmp)):
583 values = np.array(ratios[ordering[ii]][ordering[jj]])
584 values = values[np.abs(values) < 1.0]
586 calib.coeffNum[ii][jj] = len(values)
589 self.log.warn(
"No values for matrix element %d,%d" % (ii, jj))
590 calib.coeffs[ii][jj] = np.nan
591 calib.coeffErr[ii][jj] = np.nan
592 calib.coeffValid[ii][jj] =
False
595 for rej
in range(rejIter):
596 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
597 sigma = 0.741*(hi - lo)
598 good = np.abs(values - med) < rejSigma*sigma
599 if good.sum() == len(good):
601 values = values[good]
603 calib.coeffs[ii][jj] = np.mean(values)
604 if calib.coeffNum[ii][jj] == 1:
605 calib.coeffErr[ii][jj] = np.nan
608 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
609 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj])
610 > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
612 if calib.coeffNum[ii][jj] > 1:
613 self.
debugRatiosdebugRatios(
'measure', ratios, ordering[ii], ordering[jj],
614 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
620 """Apply valid constraints to the measured values.
622 Any measured coefficient that is determined to be invalid is
623 set to zero, and has the error set to nan. The validation is
624 determined by checking that the measured coefficient is larger
625 than the calculated standard error of the mean.
629 inCalib : `lsst.ip.isr.CrosstalkCalib`
630 Input calibration to filter.
634 outCalib : `lsst.ip.isr.CrosstalkCalib`
635 Filtered calibration.
637 outCalib = CrosstalkCalib()
638 outCalib.numAmps = inCalib.numAmps
640 outCalib.coeffs = inCalib.coeffs
641 outCalib.coeffs[~inCalib.coeffValid] = 0.0
643 outCalib.coeffErr = inCalib.coeffErr
644 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
646 outCalib.coeffNum = inCalib.coeffNum
647 outCalib.coeffValid = inCalib.coeffValid
651 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
652 """Utility function to examine the final CT ratio set.
657 State of processing to view.
658 ratios : `dict` of `dict` of `np.ndarray`
659 Array of measured CT ratios, indexed by source/victim
662 Index of the source amplifier.
664 Index of the target amplifier.
665 coeff : `float`, optional
666 Coefficient calculated to plot along with the simple mean.
667 valid : `bool`, optional
668 Validity to be added to the plot title.
670 frame = getDebugFrame(self._display, stepname)
672 if i == j
or ratios
is None or len(ratios) < 1:
675 ratioList = ratios[i][j]
676 if ratioList
is None or len(ratioList) < 1:
679 mean = np.mean(ratioList)
680 std = np.std(ratioList)
681 import matplotlib.pyplot
as plt
682 figure = plt.figure(1)
684 plt.hist(x=ratioList, bins=len(ratioList),
685 cumulative=
True, color=
'b', density=
True, histtype=
'step')
686 plt.xlabel(
"Measured pixel ratio")
687 plt.ylabel(f
"CDF: n={len(ratioList)}")
688 plt.xlim(np.percentile(ratioList, [1.0, 99]))
689 plt.axvline(x=mean, color=
"k")
690 plt.axvline(x=coeff, color=
'g')
691 plt.axvline(x=(std / np.sqrt(len(ratioList))), color=
'r')
692 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color=
'r')
693 plt.title(f
"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
696 prompt =
"Press Enter to continue: "
698 ans = input(prompt).lower()
699 if ans
in (
"",
"c",):
701 elif ans
in (
"pdb",
"p",):
708 extract = ConfigurableField(
709 target=CrosstalkExtractTask,
710 doc=
"Task to measure pixel ratios.",
712 solver = ConfigurableField(
713 target=CrosstalkSolveTask,
714 doc=
"Task to convert ratio lists to crosstalk coefficients.",
719 """Measure intra-detector crosstalk.
723 lsst.ip.isr.crosstalk.CrosstalkCalib
724 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
725 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
729 The crosstalk this method measures assumes that when a bright
730 pixel is found in one detector amplifier, all other detector
731 amplifiers may see a signal change in the same pixel location
732 (relative to the readout amplifier) as these other pixels are read
733 out at the same time.
735 After processing each input exposure through a limited set of ISR
736 stages, bright unmasked pixels above the threshold are identified.
737 The potential CT signal is found by taking the ratio of the
738 appropriate background-subtracted pixel value on the other
739 amplifiers to the input value on the source amplifier. If the
740 source amplifier has a large number of bright pixels as well, the
741 background level may be elevated, leading to poor ratio
744 The set of ratios found between each pair of amplifiers across all
745 input exposures is then gathered to produce the final CT
746 coefficients. The sigma-clipped mean and sigma are returned from
747 these sets of ratios, with the coefficient to supply to the ISR
748 CrosstalkTask() being the multiplicative inverse of these values.
750 This Task simply calls the pipetask versions of the measure
753 ConfigClass = MeasureCrosstalkConfig
754 _DefaultName =
"measureCrosstalk"
757 RunnerClass = DataRefListRunner
761 self.makeSubtask(
"extract")
762 self.makeSubtask(
"solver")
765 """Run extract task on each of inputs in the dataRef list, then pass
766 that to the solver task.
770 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
771 Data references for exposures for detectors to process.
775 results : `lsst.pipe.base.Struct`
776 The results struct containing:
778 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
779 Final crosstalk calibration.
780 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
781 Provenance data for the new calibration.
786 Raised if multiple target detectors are supplied.
788 dataRef = dataRefList[0]
789 camera = dataRef.get(
"camera")
793 for dataRef
in dataRefList:
794 exposure = dataRef.get(
"postISRCCD")
796 if exposure.getDetector().getName() != activeChip:
797 raise RuntimeError(
"Too many input detectors supplied!")
799 activeChip = exposure.getDetector().getName()
801 self.extract.debugView(
"extract", exposure)
802 result = self.extract.run(exposure)
803 ratios.append(result.outputRatios)
805 for detIter, detector
in enumerate(camera):
806 if detector.getName() == activeChip:
808 outputDims = {
'instrument': camera.getName(),
809 'detector': detectorId,
812 finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims)
813 dataRef.put(finalResults.outputCrosstalk,
"crosstalk")
def __init__(self, *config=None)
def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None)
def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False)
def filterCrosstalkCalib(inCalib)
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def __init__(self, **kwargs)
def runDataRef(self, dataRefList)
def sigmaClipCorrection(nSigClip)