24 from collections
import defaultdict
29 from lsstDebug
import getDebugFrame
33 from lsst.ip.isr import CrosstalkCalib, IsrProvenance
34 from lsst.pipe.tasks.getRepositoryData
import DataRefListRunner
37 from ._lookupStaticCalibration
import lookupStaticCalibration
39 __all__ = [
"CrosstalkExtractConfig",
"CrosstalkExtractTask",
40 "CrosstalkSolveTask",
"CrosstalkSolveConfig",
41 "MeasureCrosstalkConfig",
"MeasureCrosstalkTask"]
45 dimensions=(
"instrument",
"exposure",
"detector")):
47 name=
"crosstalkInputs",
48 doc=
"Input post-ISR processed exposure to measure crosstalk from.",
49 storageClass=
"Exposure",
50 dimensions=(
"instrument",
"exposure",
"detector"),
55 name=
"crosstalkSource",
56 doc=
"Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.",
57 storageClass=
"Exposure",
58 dimensions=(
"instrument",
"exposure",
"detector"),
64 outputRatios = cT.Output(
65 name=
"crosstalkRatios",
66 doc=
"Extracted crosstalk pixel ratios.",
67 storageClass=
"StructuredDataDict",
68 dimensions=(
"instrument",
"exposure",
"detector"),
70 outputFluxes = cT.Output(
71 name=
"crosstalkFluxes",
72 doc=
"Source pixel fluxes used in ratios.",
73 storageClass=
"StructuredDataDict",
74 dimensions=(
"instrument",
"exposure",
"detector"),
81 self.inputs.discard(
"sourceExp")
85 pipelineConnections=CrosstalkExtractConnections):
86 """Configuration for the measurement of pixel ratios.
88 doMeasureInterchip = Field(
91 doc=
"Measure inter-chip crosstalk as well?",
96 doc=
"Minimum level of source pixels for which to measure crosstalk."
98 ignoreSaturatedPixels = Field(
101 doc=
"Should saturated pixels be ignored?"
105 default=[
"BAD",
"INTRP"],
106 doc=
"Mask planes to ignore when identifying source pixels."
111 doc=
"Is the input exposure trimmed?"
120 if 'SAT' not in self.
badMaskbadMask:
121 self.
badMaskbadMask.append(
'SAT')
123 if 'SAT' in self.
badMaskbadMask:
124 self.
badMaskbadMask = [mask
for mask
in self.
badMaskbadMask
if mask !=
'SAT']
128 pipeBase.CmdLineTask):
129 """Task to measure pixel ratios to find crosstalk.
131 ConfigClass = CrosstalkExtractConfig
132 _DefaultName =
'cpCrosstalkExtract'
134 def run(self, inputExp, sourceExps=[]):
135 """Measure pixel ratios between amplifiers in inputExp.
137 Extract crosstalk ratios between different amplifiers.
139 For pixels above ``config.threshold``, we calculate the ratio
140 between each background-subtracted target amp and the source
141 amp. We return a list of ratios for each pixel for each
142 target/source combination, as nested dictionary containing the
147 inputExp : `lsst.afw.image.Exposure`
148 Input exposure to measure pixel ratios on.
149 sourceExp : `list` [`lsst.afw.image.Exposure`], optional
150 List of chips to use as sources to measure inter-chip
155 results : `lsst.pipe.base.Struct`
156 The results struct containing:
158 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
159 A catalog of ratio lists. The dictionaries are
161 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
162 contains the ratio list for that combination.
163 ``outputFluxes`` : `dict` [`dict` [`list`]]
164 A catalog of flux lists. The dictionaries are
166 outputFluxes[sourceChip][sourceAmp]
167 contains the flux list used in the outputRatios.
171 The lsstDebug.Info() method can be rewritten for __name__ =
172 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
174 debug.display['extract'] : `bool`
175 Display the exposure under consideration, with the pixels used
176 for crosstalk measurement indicated by the DETECTED mask plane.
177 debug.display['pixels'] : `bool`
178 Display a plot of the ratio calculated for each pixel used in this
179 exposure, split by amplifier pairs. The median value is listed
182 outputRatios = defaultdict(
lambda: defaultdict(dict))
183 outputFluxes = defaultdict(
lambda: defaultdict(dict))
185 threshold = self.config.threshold
186 badPixels = list(self.config.badMask)
188 targetDetector = inputExp.getDetector()
189 targetChip = targetDetector.getName()
192 sourceExtractExps = [inputExp]
193 sourceExtractExps.extend(sourceExps)
195 self.log.info(
"Measuring full detector background for target: %s", targetChip)
196 targetIm = inputExp.getMaskedImage()
198 detected = targetIm.getMask().getPlaneBitMask(
"DETECTED")
199 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + [
"DETECTED"])
201 self.
debugViewdebugView(
'extract', inputExp)
203 for sourceExp
in sourceExtractExps:
204 sourceDetector = sourceExp.getDetector()
205 sourceChip = sourceDetector.getName()
206 sourceIm = sourceExp.getMaskedImage()
207 bad = sourceIm.getMask().getPlaneBitMask(badPixels)
208 self.log.info(
"Measuring crosstalk from source: %s", sourceChip)
210 if sourceExp != inputExp:
212 detected = sourceIm.getMask().getPlaneBitMask(
"DETECTED")
215 ratioDict = defaultdict(
lambda: defaultdict(list))
218 for sourceAmp
in sourceDetector:
219 sourceAmpName = sourceAmp.getName()
220 sourceAmpImage = sourceIm[sourceAmp.getBBox()]
221 sourceMask = sourceAmpImage.mask.array
222 select = ((sourceMask & detected > 0)
223 & (sourceMask & bad == 0)
224 & np.isfinite(sourceAmpImage.image.array))
225 count = np.sum(select)
226 self.log.debug(
" Source amplifier: %s", sourceAmpName)
228 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
230 for targetAmp
in targetDetector:
232 targetAmpName = targetAmp.getName()
233 if sourceAmpName == targetAmpName
and sourceChip == targetChip:
234 ratioDict[sourceAmpName][targetAmpName] = []
236 self.log.debug(
" Target amplifier: %s", targetAmpName)
238 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
239 targetAmp, sourceAmp,
240 isTrimmed=self.config.isTrimmed)
241 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
242 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
243 extractedCount += count
246 sourceAmpImage.image.array[select],
247 targetAmpImage.array[select] - bg,
248 sourceAmpName, targetAmpName)
250 self.log.info(
"Extracted %d pixels from %s -> %s (targetBG: %f)",
251 extractedCount, sourceChip, targetChip, bg)
252 outputRatios[targetChip][sourceChip] = ratioDict
254 return pipeBase.Struct(
260 """Utility function to examine the image being processed.
265 State of processing to view.
266 exposure : `lsst.afw.image.Exposure`
269 frame = getDebugFrame(self._display, stepname)
271 display = getDisplay(frame)
272 display.scale(
'asinh',
'zscale')
273 display.mtv(exposure)
275 prompt =
"Press Enter to continue: "
277 ans = input(prompt).lower()
278 if ans
in (
"",
"c",):
281 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName):
282 """Utility function to examine the CT ratio pixel values.
287 State of processing to view.
288 pixelsIn : `np.ndarray`
289 Pixel values from the potential crosstalk source.
290 pixelsOut : `np.ndarray`
291 Pixel values from the potential crosstalk target.
293 Source amplifier name
295 Target amplifier name
297 frame = getDebugFrame(self._display, stepname)
299 import matplotlib.pyplot
as plt
300 figure = plt.figure(1)
303 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
304 axes.plot(pixelsIn, pixelsOut / pixelsIn,
'k+')
305 plt.xlabel(
"Source amplifier pixel value")
306 plt.ylabel(
"Measured pixel ratio")
307 plt.title(f
"(Source {sourceName} -> Target {targetName}) median ratio: "
308 f
"{(np.median(pixelsOut / pixelsIn))}")
311 prompt =
"Press Enter to continue: "
313 ans = input(prompt).lower()
314 if ans
in (
"",
"c",):
320 dimensions=(
"instrument",
"detector")):
321 inputRatios = cT.Input(
322 name=
"crosstalkRatios",
323 doc=
"Ratios measured for an input exposure.",
324 storageClass=
"StructuredDataDict",
325 dimensions=(
"instrument",
"exposure",
"detector"),
328 inputFluxes = cT.Input(
329 name=
"crosstalkFluxes",
330 doc=
"Fluxes of CT source pixels, for nonlinear fits.",
331 storageClass=
"StructuredDataDict",
332 dimensions=(
"instrument",
"exposure",
"detector"),
335 camera = cT.PrerequisiteInput(
337 doc=
"Camera the input data comes from.",
338 storageClass=
"Camera",
339 dimensions=(
"instrument",),
341 lookupFunction=lookupStaticCalibration,
344 outputCrosstalk = cT.Output(
346 doc=
"Output proposed crosstalk calibration.",
347 storageClass=
"CrosstalkCalib",
348 dimensions=(
"instrument",
"detector"),
356 if config.fluxOrder == 0:
357 self.inputs.discard(
"inputFluxes")
361 pipelineConnections=CrosstalkSolveConnections):
362 """Configuration for the solving of crosstalk from pixel ratios.
367 doc=
"Number of rejection iterations for final coefficient calculation.",
372 doc=
"Rejection threshold (sigma) for final coefficient calculation.",
377 doc=
"Polynomial order in source flux to fit crosstalk.",
382 doc=
"Filter generated crosstalk to remove marginal measurements.",
387 pipeBase.CmdLineTask):
388 """Task to solve crosstalk from pixel ratios.
390 ConfigClass = CrosstalkSolveConfig
391 _DefaultName =
'cpCrosstalkSolve'
394 """Ensure that the input and output dimensions are passed along.
398 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext`
399 Butler to operate on.
400 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection`
401 Input data refs to load.
402 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection`
403 Output data refs to persist.
405 inputs = butlerQC.get(inputRefs)
408 inputs[
'inputDims'] = [exp.dataId.byName()
for exp
in inputRefs.inputRatios]
409 inputs[
'outputDims'] = outputRefs.outputCrosstalk.dataId.byName()
411 outputs = self.
runrun(**inputs)
412 butlerQC.put(outputs, outputRefs)
414 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None):
415 """Combine ratios to produce crosstalk coefficients.
419 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]]
420 A list of nested dictionaries of ratios indexed by target
421 and source chip, then by target and source amplifier.
422 inputFluxes : `list` [`dict` [`dict` [`list`]]]
423 A list of nested dictionaries of source pixel fluxes, indexed
424 by source chip and amplifier.
425 camera : `lsst.afw.cameraGeom.Camera`
427 inputDims : `list` [`lsst.daf.butler.DataCoordinate`]
428 DataIds to use to construct provenance.
429 outputDims : `list` [`lsst.daf.butler.DataCoordinate`]
430 DataIds to use to populate the output calibration.
434 results : `lsst.pipe.base.Struct`
435 The results struct containing:
437 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
438 Final crosstalk calibration.
439 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
440 Provenance data for the new calibration.
445 Raised if the input data contains multiple target detectors.
449 The lsstDebug.Info() method can be rewritten for __name__ =
450 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
452 debug.display['reduce'] : `bool`
453 Display a histogram of the combined ratio measurements for
454 a pair of source/target amplifiers from all input
459 calibChip = outputDims[
'detector']
460 instrument = outputDims[
'instrument']
466 self.log.info(
"Combining measurements from %d ratios and %d fluxes",
467 len(inputRatios), len(inputFluxes)
if inputFluxes
else 0)
469 if inputFluxes
is None:
470 inputFluxes = [
None for exp
in inputRatios]
472 combinedRatios = defaultdict(
lambda: defaultdict(list))
473 combinedFluxes = defaultdict(
lambda: defaultdict(list))
474 for ratioDict, fluxDict
in zip(inputRatios, inputFluxes):
475 for targetChip
in ratioDict:
476 if calibChip
and targetChip != calibChip:
477 raise RuntimeError(
"Received multiple target chips!")
479 sourceChip = targetChip
480 if sourceChip
in ratioDict[targetChip]:
481 ratios = ratioDict[targetChip][sourceChip]
483 for targetAmp
in ratios:
484 for sourceAmp
in ratios[targetAmp]:
485 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp])
487 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp])
492 for targetAmp
in combinedRatios:
493 for sourceAmp
in combinedRatios[targetAmp]:
494 self.log.info(
"Read %d pixels for %s -> %s",
495 len(combinedRatios[targetAmp][sourceAmp]),
496 targetAmp, sourceAmp)
497 if len(combinedRatios[targetAmp][sourceAmp]) > 1:
498 self.
debugRatiosdebugRatios(
'reduce', combinedRatios, targetAmp, sourceAmp)
500 if self.config.fluxOrder == 0:
501 self.log.info(
"Fitting crosstalk coefficients.")
503 self.config.rejIter, self.config.rejSigma)
505 raise NotImplementedError(
"Non-linear crosstalk terms are not yet supported.")
507 self.log.info(
"Number of valid coefficients: %d", np.sum(calib.coeffValid))
509 if self.config.doFiltering:
512 self.log.info(
"Filtering measured crosstalk to remove invalid solutions.")
516 calib.hasCrosstalk =
True
520 calib._detectorId = calibChip
522 calib._detectorName = camera[calibChip].getName()
523 calib._detectorSerial = camera[calibChip].getSerial()
525 calib._instrument = instrument
526 calib.updateMetadata(setCalibId=
True, setDate=
True)
529 provenance = IsrProvenance(calibType=
"CROSSTALK")
530 provenance._detectorName = calibChip
532 provenance.fromDataIds(inputDims)
533 provenance._instrument = instrument
534 provenance.updateMetadata()
536 return pipeBase.Struct(
537 outputCrosstalk=calib,
538 outputProvenance=provenance,
542 """Measure crosstalk coefficients from the ratios.
544 Given a list of ratios for each target/source amp combination,
545 we measure a sigma clipped mean and error.
547 The coefficient errors returned are the standard deviation of
548 the final set of clipped input ratios.
552 ratios : `dict` of `dict` of `numpy.ndarray`
553 Catalog of arrays of ratios.
555 Number of rejection iterations.
557 Rejection threshold (sigma).
561 calib : `lsst.ip.isr.CrosstalkCalib`
562 The output crosstalk calibration.
566 The lsstDebug.Info() method can be rewritten for __name__ =
567 `lsst.ip.isr.measureCrosstalk`, and supports the parameters:
569 debug.display['measure'] : `bool`
570 Display the CDF of the combined ratio measurements for
571 a pair of source/target amplifiers from the final set of
572 clipped input ratios.
574 calib = CrosstalkCalib(nAmp=len(ratios))
577 ordering = list(ratios.keys())
578 for ii, jj
in itertools.product(range(calib.nAmp), range(calib.nAmp)):
582 values = np.array(ratios[ordering[ii]][ordering[jj]])
583 values = values[np.abs(values) < 1.0]
585 calib.coeffNum[ii][jj] = len(values)
588 self.log.warn(
"No values for matrix element %d,%d" % (ii, jj))
589 calib.coeffs[ii][jj] = np.nan
590 calib.coeffErr[ii][jj] = np.nan
591 calib.coeffValid[ii][jj] =
False
594 for rej
in range(rejIter):
595 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
596 sigma = 0.741*(hi - lo)
597 good = np.abs(values - med) < rejSigma*sigma
598 if good.sum() == len(good):
600 values = values[good]
602 calib.coeffs[ii][jj] = np.mean(values)
603 if calib.coeffNum[ii][jj] == 1:
604 calib.coeffErr[ii][jj] = np.nan
607 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
608 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj])
609 > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
611 if calib.coeffNum[ii][jj] > 1:
612 self.
debugRatiosdebugRatios(
'measure', ratios, ordering[ii], ordering[jj],
613 calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
619 """Apply valid constraints to the measured values.
621 Any measured coefficient that is determined to be invalid is
622 set to zero, and has the error set to nan. The validation is
623 determined by checking that the measured coefficient is larger
624 than the calculated standard error of the mean.
628 inCalib : `lsst.ip.isr.CrosstalkCalib`
629 Input calibration to filter.
633 outCalib : `lsst.ip.isr.CrosstalkCalib`
634 Filtered calibration.
636 outCalib = CrosstalkCalib()
637 outCalib.numAmps = inCalib.numAmps
639 outCalib.coeffs = inCalib.coeffs
640 outCalib.coeffs[~inCalib.coeffValid] = 0.0
642 outCalib.coeffErr = inCalib.coeffErr
643 outCalib.coeffErr[~inCalib.coeffValid] = np.nan
645 outCalib.coeffNum = inCalib.coeffNum
646 outCalib.coeffValid = inCalib.coeffValid
650 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False):
651 """Utility function to examine the final CT ratio set.
656 State of processing to view.
657 ratios : `dict` of `dict` of `np.ndarray`
658 Array of measured CT ratios, indexed by source/victim
661 Index of the source amplifier.
663 Index of the target amplifier.
664 coeff : `float`, optional
665 Coefficient calculated to plot along with the simple mean.
666 valid : `bool`, optional
667 Validity to be added to the plot title.
669 frame = getDebugFrame(self._display, stepname)
671 if i == j
or ratios
is None or len(ratios) < 1:
674 ratioList = ratios[i][j]
675 if ratioList
is None or len(ratioList) < 1:
678 mean = np.mean(ratioList)
679 std = np.std(ratioList)
680 import matplotlib.pyplot
as plt
681 figure = plt.figure(1)
683 plt.hist(x=ratioList, bins=len(ratioList),
684 cumulative=
True, color=
'b', density=
True, histtype=
'step')
685 plt.xlabel(
"Measured pixel ratio")
686 plt.ylabel(f
"CDF: n={len(ratioList)}")
687 plt.xlim(np.percentile(ratioList, [1.0, 99]))
688 plt.axvline(x=mean, color=
"k")
689 plt.axvline(x=coeff, color=
'g')
690 plt.axvline(x=(std / np.sqrt(len(ratioList))), color=
'r')
691 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color=
'r')
692 plt.title(f
"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}")
695 prompt =
"Press Enter to continue: "
697 ans = input(prompt).lower()
698 if ans
in (
"",
"c",):
700 elif ans
in (
"pdb",
"p",):
707 extract = ConfigurableField(
708 target=CrosstalkExtractTask,
709 doc=
"Task to measure pixel ratios.",
711 solver = ConfigurableField(
712 target=CrosstalkSolveTask,
713 doc=
"Task to convert ratio lists to crosstalk coefficients.",
718 """Measure intra-detector crosstalk.
722 lsst.ip.isr.crosstalk.CrosstalkCalib
723 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask
724 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask
728 The crosstalk this method measures assumes that when a bright
729 pixel is found in one detector amplifier, all other detector
730 amplifiers may see a signal change in the same pixel location
731 (relative to the readout amplifier) as these other pixels are read
732 out at the same time.
734 After processing each input exposure through a limited set of ISR
735 stages, bright unmasked pixels above the threshold are identified.
736 The potential CT signal is found by taking the ratio of the
737 appropriate background-subtracted pixel value on the other
738 amplifiers to the input value on the source amplifier. If the
739 source amplifier has a large number of bright pixels as well, the
740 background level may be elevated, leading to poor ratio
743 The set of ratios found between each pair of amplifiers across all
744 input exposures is then gathered to produce the final CT
745 coefficients. The sigma-clipped mean and sigma are returned from
746 these sets of ratios, with the coefficient to supply to the ISR
747 CrosstalkTask() being the multiplicative inverse of these values.
749 This Task simply calls the pipetask versions of the measure
752 ConfigClass = MeasureCrosstalkConfig
753 _DefaultName =
"measureCrosstalk"
756 RunnerClass = DataRefListRunner
760 self.makeSubtask(
"extract")
761 self.makeSubtask(
"solver")
764 """Run extract task on each of inputs in the dataRef list, then pass
765 that to the solver task.
769 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`]
770 Data references for exposures for detectors to process.
774 results : `lsst.pipe.base.Struct`
775 The results struct containing:
777 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib`
778 Final crosstalk calibration.
779 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
780 Provenance data for the new calibration.
785 Raised if multiple target detectors are supplied.
787 dataRef = dataRefList[0]
788 camera = dataRef.get(
"camera")
792 for dataRef
in dataRefList:
793 exposure = dataRef.get(
"postISRCCD")
795 if exposure.getDetector().getName() != activeChip:
796 raise RuntimeError(
"Too many input detectors supplied!")
798 activeChip = exposure.getDetector().getName()
800 self.extract.debugView(
"extract", exposure)
801 result = self.extract.run(exposure)
802 ratios.append(result.outputRatios)
804 for detIter, detector
in enumerate(camera):
805 if detector.getName() == activeChip:
807 outputDims = {
'instrument': camera.getName(),
808 'detector': detectorId,
811 finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims)
812 dataRef.put(finalResults.outputCrosstalk,
"crosstalk")
def __init__(self, *config=None)
def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None)
def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False)
def filterCrosstalkCalib(inCalib)
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def __init__(self, **kwargs)
def runDataRef(self, dataRefList)
def sigmaClipCorrection(nSigClip)