Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import itertools 

22import numpy as np 

23from scipy.stats import norm 

24 

25from collections import defaultdict 

26 

27import lsst.pipe.base as pipeBase 

28import lsst.pipe.base.connectionTypes as cT 

29 

30from lsstDebug import getDebugFrame 

31from lsst.afw.detection import FootprintSet, Threshold 

32from lsst.afw.display import getDisplay 

33from lsst.pex.config import Config, Field, ListField, ConfigurableField 

34from lsst.ip.isr import CrosstalkCalib, IsrProvenance 

35from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

36 

37__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask", 

38 "CrosstalkSolveTask", "CrosstalkSolveConfig", 

39 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"] 

40 

41 

42class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "exposure", "detector")): 

44 inputExp = cT.Input( 

45 name="crosstalkInputs", 

46 doc="Input post-ISR processed exposure to measure crosstalk from.", 

47 storageClass="Exposure", 

48 dimensions=("instrument", "exposure", "detector"), 

49 multiple=False, 

50 ) 

51 # TODO: Depends on DM-21904. 

52 sourceExp = cT.Input( 

53 name="crosstalkSource", 

54 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.", 

55 storageClass="Exposure", 

56 dimensions=("instrument", "exposure", "detector"), 

57 multiple=True, 

58 deferLoad=True, 

59 # lookupFunction=None, 

60 ) 

61 

62 outputRatios = cT.Output( 

63 name="crosstalkRatios", 

64 doc="Extracted crosstalk pixel ratios.", 

65 storageClass="StructuredDataDict", 

66 dimensions=("instrument", "exposure", "detector"), 

67 ) 

68 outputFluxes = cT.Output( 

69 name="crosstalkFluxes", 

70 doc="Source pixel fluxes used in ratios.", 

71 storageClass="StructuredDataDict", 

72 dimensions=("instrument", "exposure", "detector"), 

73 ) 

74 

75 def __init__(self, *, config=None): 

76 super().__init__(config=config) 

77 # Discard sourceExp until DM-21904 allows full interchip 

78 # measurements. 

79 self.inputs.discard("sourceExp") 

80 

81 

82class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig, 

83 pipelineConnections=CrosstalkExtractConnections): 

84 """Configuration for the measurement of pixel ratios. 

85 """ 

86 doMeasureInterchip = Field( 

87 dtype=bool, 

88 default=False, 

89 doc="Measure inter-chip crosstalk as well?", 

90 ) 

91 threshold = Field( 

92 dtype=float, 

93 default=30000, 

94 doc="Minimum level of source pixels for which to measure crosstalk." 

95 ) 

96 ignoreSaturatedPixels = Field( 

97 dtype=bool, 

98 default=True, 

99 doc="Should saturated pixels be ignored?" 

100 ) 

101 badMask = ListField( 

102 dtype=str, 

103 default=["BAD", "INTRP"], 

104 doc="Mask planes to ignore when identifying source pixels." 

105 ) 

106 isTrimmed = Field( 

107 dtype=bool, 

108 default=True, 

109 doc="Is the input exposure trimmed?" 

110 ) 

111 

112 def validate(self): 

113 super().validate() 

114 

115 # Ensure the handling of the SAT mask plane is consistent 

116 # with the ignoreSaturatedPixels value. 

117 if self.ignoreSaturatedPixels: 

118 if 'SAT' not in self.badMask: 

119 self.badMask.append('SAT') 

120 else: 

121 if 'SAT' in self.badMask: 

122 self.badMask = [mask for mask in self.badMask if mask != 'SAT'] 

123 

124 

125class CrosstalkExtractTask(pipeBase.PipelineTask, 

126 pipeBase.CmdLineTask): 

127 """Task to measure pixel ratios to find crosstalk. 

128 """ 

129 ConfigClass = CrosstalkExtractConfig 

130 _DefaultName = 'cpCrosstalkExtract' 

131 

132 def run(self, inputExp, sourceExps=[]): 

133 """Measure pixel ratios between amplifiers in inputExp. 

134 

135 Extract crosstalk ratios between different amplifiers. 

136 

137 For pixels above ``config.threshold``, we calculate the ratio 

138 between each background-subtracted target amp and the source 

139 amp. We return a list of ratios for each pixel for each 

140 target/source combination, as nested dictionary containing the 

141 ratio. 

142 

143 Parameters 

144 ---------- 

145 inputExp : `lsst.afw.image.Exposure` 

146 Input exposure to measure pixel ratios on. 

147 sourceExp : `list` [`lsst.afw.image.Exposure`], optional 

148 List of chips to use as sources to measure inter-chip 

149 crosstalk. 

150 

151 Returns 

152 ------- 

153 results : `lsst.pipe.base.Struct` 

154 The results struct containing: 

155 

156 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]] 

157 A catalog of ratio lists. The dictionaries are 

158 indexed such that: 

159 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp] 

160 contains the ratio list for that combination. 

161 ``outputFluxes`` : `dict` [`dict` [`list`]] 

162 A catalog of flux lists. The dictionaries are 

163 indexed such that: 

164 outputFluxes[sourceChip][sourceAmp] 

165 contains the flux list used in the outputRatios. 

166 

167 Notes 

168 ----- 

169 The lsstDebug.Info() method can be rewritten for __name__ = 

170 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters: 

171 

172 debug.display['extract'] : `bool` 

173 Display the exposure under consideration, with the pixels used 

174 for crosstalk measurement indicated by the DETECTED mask plane. 

175 debug.display['pixels'] : `bool` 

176 Display a plot of the ratio calculated for each pixel used in this 

177 exposure, split by amplifier pairs. The median value is listed 

178 for reference. 

179 """ 

180 outputRatios = defaultdict(lambda: defaultdict(dict)) 

181 outputFluxes = defaultdict(lambda: defaultdict(dict)) 

182 

183 threshold = self.config.threshold 

184 badPixels = list(self.config.badMask) 

185 

186 targetDetector = inputExp.getDetector() 

187 targetChip = targetDetector.getName() 

188 

189 # Always look at the target chip first, then go to any other supplied exposures. 

190 sourceExtractExps = [inputExp] 

191 sourceExtractExps.extend(sourceExps) 

192 

193 self.log.info("Measuring full detector background for target: %s", targetChip) 

194 targetIm = inputExp.getMaskedImage() 

195 FootprintSet(targetIm, Threshold(threshold), "DETECTED") 

196 detected = targetIm.getMask().getPlaneBitMask("DETECTED") 

197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) 

198 

199 self.debugView('extract', inputExp) 

200 

201 for sourceExp in sourceExtractExps: 

202 sourceDetector = sourceExp.getDetector() 

203 sourceChip = sourceDetector.getName() 

204 sourceIm = sourceExp.getMaskedImage() 

205 bad = sourceIm.getMask().getPlaneBitMask(badPixels) 

206 self.log.info("Measuring crosstalk from source: %s", sourceChip) 

207 

208 if sourceExp != inputExp: 

209 FootprintSet(sourceIm, Threshold(threshold), "DETECTED") 

210 detected = sourceIm.getMask().getPlaneBitMask("DETECTED") 

211 

212 # The dictionary of amp-to-amp ratios for this pair of source->target detectors. 

213 ratioDict = defaultdict(lambda: defaultdict(list)) 

214 extractedCount = 0 

215 

216 for sourceAmp in sourceDetector: 

217 sourceAmpName = sourceAmp.getName() 

218 sourceAmpImage = sourceIm[sourceAmp.getBBox()] 

219 sourceMask = sourceAmpImage.mask.array 

220 select = ((sourceMask & detected > 0) & 

221 (sourceMask & bad == 0) & 

222 np.isfinite(sourceAmpImage.image.array)) 

223 count = np.sum(select) 

224 self.log.debug(" Source amplifier: %s", sourceAmpName) 

225 

226 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select] 

227 

228 for targetAmp in targetDetector: 

229 # iterate over targetExposure 

230 targetAmpName = targetAmp.getName() 

231 if sourceAmpName == targetAmpName and sourceChip == targetChip: 

232 ratioDict[sourceAmpName][targetAmpName] = [] 

233 continue 

234 self.log.debug(" Target amplifier: %s", targetAmpName) 

235 

236 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image, 

237 targetAmp, sourceAmp, 

238 isTrimmed=self.config.isTrimmed) 

239 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select] 

240 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist() 

241 extractedCount += count 

242 

243 self.debugPixels('pixels', 

244 sourceAmpImage.image.array[select], 

245 targetAmpImage.array[select] - bg, 

246 sourceAmpName, targetAmpName) 

247 

248 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)", 

249 extractedCount, sourceChip, targetChip, bg) 

250 outputRatios[targetChip][sourceChip] = ratioDict 

251 

252 return pipeBase.Struct( 

253 outputRatios=outputRatios, 

254 outputFluxes=outputFluxes 

255 ) 

256 

257 def debugView(self, stepname, exposure): 

258 """Utility function to examine the image being processed. 

259 

260 Parameters 

261 ---------- 

262 stepname : `str` 

263 State of processing to view. 

264 exposure : `lsst.afw.image.Exposure` 

265 Exposure to view. 

266 """ 

267 frame = getDebugFrame(self._display, stepname) 

268 if frame: 

269 display = getDisplay(frame) 

270 display.scale('asinh', 'zscale') 

271 display.mtv(exposure) 

272 

273 prompt = "Press Enter to continue: " 

274 while True: 

275 ans = input(prompt).lower() 

276 if ans in ("", "c",): 

277 break 

278 

279 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName): 

280 """Utility function to examine the CT ratio pixel values. 

281 

282 Parameters 

283 ---------- 

284 stepname : `str` 

285 State of processing to view. 

286 pixelsIn : `np.ndarray` 

287 Pixel values from the potential crosstalk source. 

288 pixelsOut : `np.ndarray` 

289 Pixel values from the potential crosstalk target. 

290 sourceName : `str` 

291 Source amplifier name 

292 targetName : `str` 

293 Target amplifier name 

294 """ 

295 frame = getDebugFrame(self._display, stepname) 

296 if frame: 

297 import matplotlib.pyplot as plt 

298 figure = plt.figure(1) 

299 figure.clear() 

300 

301 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

302 axes.plt(pixelsIn, pixelsOut / pixelsIn, 'k+') 

303 plt.xlabel("Source amplifier pixel value") 

304 plt.ylabel("Measured pixel ratio") 

305 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: " 

306 f"{(np.median(pixelsOut / pixelsIn))}") 

307 figure.show() 

308 

309 prompt = "Press Enter to continue: " 

310 while True: 

311 ans = input(prompt).lower() 

312 if ans in ("", "c",): 

313 break 

314 plt.close() 

315 

316 

317class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections, 

318 dimensions=("instrument", "detector")): 

319 inputRatios = cT.Input( 

320 name="crosstalkRatios", 

321 doc="Ratios measured for an input exposure.", 

322 storageClass="StructuredDataDict", 

323 dimensions=("instrument", "exposure", "detector"), 

324 multiple=True, 

325 ) 

326 inputFluxes = cT.Input( 

327 name="crosstalkFluxes", 

328 doc="Fluxes of CT source pixels, for nonlinear fits.", 

329 storageClass="StructuredDataDict", 

330 dimensions=("instrument", "exposure", "detector"), 

331 multiple=True, 

332 ) 

333 camera = cT.PrerequisiteInput( 

334 name="camera", 

335 doc="Camera the input data comes from.", 

336 storageClass="Camera", 

337 dimensions=("instrument", "calibration_label"), 

338 ) 

339 

340 outputCrosstalk = cT.Output( 

341 name="crosstalkProposal", 

342 doc="Output proposed crosstalk calibration.", 

343 storageClass="CrosstalkCalib", 

344 dimensions=("instrument", "detector"), 

345 multiple=False, 

346 ) 

347 

348 def __init__(self, *, config=None): 

349 super().__init__(config=config) 

350 

351 if config.fluxOrder == 0: 

352 self.Inputs.discard("inputFluxes") 

353 

354 

355class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig, 

356 pipelineConnections=CrosstalkSolveConnections): 

357 """Configuration for the solving of crosstalk from pixel ratios. 

358 """ 

359 rejIter = Field( 

360 dtype=int, 

361 default=3, 

362 doc="Number of rejection iterations for final coefficient calculation.", 

363 ) 

364 rejSigma = Field( 

365 dtype=float, 

366 default=2.0, 

367 doc="Rejection threshold (sigma) for final coefficient calculation.", 

368 ) 

369 fluxOrder = Field( 

370 dtype=int, 

371 default=0, 

372 doc="Polynomial order in source flux to fit crosstalk.", 

373 ) 

374 doFiltering = Field( 

375 dtype=bool, 

376 default=False, 

377 doc="Filter generated crosstalk to remove marginal measurements.", 

378 ) 

379 

380 

381class CrosstalkSolveTask(pipeBase.PipelineTask, 

382 pipeBase.CmdLineTask): 

383 """Task to solve crosstalk from pixel ratios. 

384 """ 

385 ConfigClass = CrosstalkSolveConfig 

386 _DefaultName = 'cpCrosstalkSolve' 

387 

388 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

389 """Ensure that the input and output dimensions are passed along. 

390 

391 Parameters 

392 ---------- 

393 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

394 Butler to operate on. 

395 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

396 Input data refs to load. 

397 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

398 Output data refs to persist. 

399 """ 

400 inputs = butlerQC.get(inputRefs) 

401 

402 # Use the dimensions to set calib/provenance information. 

403 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios] 

404 inputs['outputDims'] = [exp.dataId.byName() for exp in outputRefs.outputCrosstalk] 

405 

406 outputs = self.run(**inputs) 

407 butlerQC.put(outputs, outputRefs) 

408 

409 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None): 

410 """Combine ratios to produce crosstalk coefficients. 

411 

412 Parameters 

413 ---------- 

414 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]] 

415 A list of nested dictionaries of ratios indexed by target 

416 and source chip, then by target and source amplifier. 

417 inputFluxes : `list` [`dict` [`dict` [`list`]]] 

418 A list of nested dictionaries of source pixel fluxes, indexed 

419 by source chip and amplifier. 

420 camera : `lsst.afw.cameraGeom.Camera` 

421 Input camera. 

422 inputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

423 DataIds to use to construct provenance. 

424 outputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

425 DataIds to use to populate the output calibration. 

426 

427 Returns 

428 ------- 

429 results : `lsst.pipe.base.Struct` 

430 The results struct containing: 

431 

432 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib` 

433 Final crosstalk calibration. 

434 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

435 Provenance data for the new calibration. 

436 

437 Raises 

438 ------ 

439 RuntimeError 

440 Raised if the input data contains multiple target detectors. 

441 

442 Notes 

443 ----- 

444 The lsstDebug.Info() method can be rewritten for __name__ = 

445 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

446 

447 debug.display['reduce'] : `bool` 

448 Display a histogram of the combined ratio measurements for 

449 a pair of source/target amplifiers from all input 

450 exposures/detectors. 

451 

452 """ 

453 if outputDims: 

454 calibChip = outputDims['detector'] 

455 else: 

456 # calibChip needs to be set manually in Gen2. 

457 calibChip = None 

458 

459 self.log.info("Combining measurements from %d ratios and %d fluxes", 

460 len(inputRatios), len(inputFluxes) if inputFluxes else 0) 

461 

462 if inputFluxes is None: 

463 inputFluxes = [None for exp in inputRatios] 

464 

465 combinedRatios = defaultdict(lambda: defaultdict(list)) 

466 combinedFluxes = defaultdict(lambda: defaultdict(list)) 

467 for ratioDict, fluxDict in zip(inputRatios, inputFluxes): 

468 for targetChip in ratioDict: 

469 if calibChip and targetChip != calibChip: 

470 raise RuntimeError("Received multiple target chips!") 

471 

472 sourceChip = targetChip 

473 if sourceChip in ratioDict[targetChip]: 

474 ratios = ratioDict[targetChip][sourceChip] 

475 

476 for targetAmp in ratios: 

477 for sourceAmp in ratios[targetAmp]: 

478 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp]) 

479 if fluxDict: 

480 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp]) 

481 # TODO: DM-21904 

482 # Iterating over all other entries in ratioDict[targetChip] will yield 

483 # inter-chip terms. 

484 

485 for targetAmp in combinedRatios: 

486 for sourceAmp in combinedRatios[targetAmp]: 

487 self.log.info("Read %d pixels for %s -> %s", 

488 len(combinedRatios[targetAmp][sourceAmp]), 

489 targetAmp, sourceAmp) 

490 if len(combinedRatios[targetAmp][sourceAmp]) > 1: 

491 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp) 

492 

493 if self.config.fluxOrder == 0: 

494 self.log.info("Fitting crosstalk coefficients.") 

495 calib = self.measureCrosstalkCoefficients(combinedRatios, 

496 self.config.rejIter, self.config.rejSigma) 

497 else: 

498 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.") 

499 

500 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid)) 

501 

502 if self.config.doFiltering: 

503 # This step will apply the calculated validity values to 

504 # censor poorly measured coefficients. 

505 self.log.info("Filtering measured crosstalk to remove invalid solutions.") 

506 calib = self.filterCrosstalkCalib(calib) 

507 

508 # Populate the remainder of the calibration information. 

509 calib.hasCrosstalk = True 

510 calib.interChip = {} 

511 calib._detectorName = calibChip 

512 if camera: 

513 for chip in camera: 

514 if chip.getName() == calibChip: 

515 calib._detectorSerial = chip.getSerial() 

516 calib.updateMetadata() 

517 

518 # Make an IsrProvenance(). 

519 provenance = IsrProvenance(calibType="CROSSTALK") 

520 provenance._detectorName = calibChip 

521 if inputDims: 

522 provenance.fromDataIds(inputDims) 

523 provenance._instrument = outputDims['instrument'] 

524 provenance.updateMetadata() 

525 

526 return pipeBase.Struct( 

527 outputCrosstalk=calib, 

528 outputProvenance=provenance, 

529 ) 

530 

531 def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma): 

532 """Measure crosstalk coefficients from the ratios. 

533 

534 Given a list of ratios for each target/source amp combination, 

535 we measure a sigma clipped mean and error. 

536 

537 The coefficient errors returned are the standard deviation of 

538 the final set of clipped input ratios. 

539 

540 Parameters 

541 ---------- 

542 ratios : `dict` of `dict` of `numpy.ndarray` 

543 Catalog of arrays of ratios. 

544 rejIter : `int` 

545 Number of rejection iterations. 

546 rejSigma : `float` 

547 Rejection threshold (sigma). 

548 

549 Returns 

550 ------- 

551 calib : `lsst.ip.isr.CrosstalkCalib` 

552 The output crosstalk calibration. 

553 

554 Notes 

555 ----- 

556 The lsstDebug.Info() method can be rewritten for __name__ = 

557 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

558 

559 debug.display['measure'] : `bool` 

560 Display the CDF of the combined ratio measurements for 

561 a pair of source/target amplifiers from the final set of 

562 clipped input ratios. 

563 """ 

564 calib = CrosstalkCalib(nAmp=len(ratios)) 

565 

566 # Calibration stores coefficients as a numpy ndarray. 

567 ordering = list(ratios.keys()) 

568 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)): 

569 if ii == jj: 

570 values = [0.0] 

571 else: 

572 values = np.array(ratios[ordering[ii]][ordering[jj]]) 

573 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

574 

575 calib.coeffNum[ii][jj] = len(values) 

576 

577 if len(values) == 0: 

578 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

579 calib.coeffs[ii][jj] = np.nan 

580 calib.coeffErr[ii][jj] = np.nan 

581 calib.coeffValid[ii][jj] = False 

582 else: 

583 if ii != jj: 

584 for rej in range(rejIter): 

585 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

586 sigma = 0.741*(hi - lo) 

587 good = np.abs(values - med) < rejSigma*sigma 

588 if good.sum() == len(good): 

589 break 

590 values = values[good] 

591 

592 calib.coeffs[ii][jj] = np.mean(values) 

593 if calib.coeffNum[ii][jj] == 1: 

594 calib.coeffErr[ii][jj] = np.nan 

595 else: 

596 correctionFactor = self.sigmaClipCorrection(rejSigma) 

597 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor 

598 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) > 

599 calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj])) 

600 

601 if calib.coeffNum[ii][jj] > 1: 

602 self.debugRatios('measure', ratios, ordering[ii], ordering[jj], 

603 calib.coeffs[ii][jj], calib.coeffValid[ii][jj]) 

604 

605 return calib 

606 

607 @staticmethod 

608 def sigmaClipCorrection(nSigClip): 

609 """Correct measured sigma to account for clipping. 

610 

611 If we clip our input data and then measure sigma, then the 

612 measured sigma is smaller than the true value because real 

613 points beyond the clip threshold have been removed. This is a 

614 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the 

615 default parameters for measure crosstalk use nSigClip=2.0. 

616 This causes the measured sigma to be about 15% smaller than 

617 real. This formula corrects the issue, for the symmetric case 

618 (upper clip threshold equal to lower clip threshold). 

619 

620 Parameters 

621 ---------- 

622 nSigClip : `float` 

623 Number of sigma the measurement was clipped by. 

624 

625 Returns 

626 ------- 

627 scaleFactor : `float` 

628 Scale factor to increase the measured sigma by. 

629 

630 """ 

631 varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip)) 

632 return 1.0 / np.sqrt(varFactor) 

633 

634 @staticmethod 

635 def filterCrosstalkCalib(inCalib): 

636 """Apply valid constraints to the measured values. 

637 

638 Any measured coefficient that is determined to be invalid is 

639 set to zero, and has the error set to nan. The validation is 

640 determined by checking that the measured coefficient is larger 

641 than the calculated standard error of the mean. 

642 

643 Parameters 

644 ---------- 

645 inCalib : `lsst.ip.isr.CrosstalkCalib` 

646 Input calibration to filter. 

647 

648 Returns 

649 ------- 

650 outCalib : `lsst.ip.isr.CrosstalkCalib` 

651 Filtered calibration. 

652 """ 

653 outCalib = CrosstalkCalib() 

654 outCalib.numAmps = inCalib.numAmps 

655 

656 outCalib.coeffs = inCalib.coeffs 

657 outCalib.coeffs[~inCalib.coeffValid] = 0.0 

658 

659 outCalib.coeffErr = inCalib.coeffErr 

660 outCalib.coeffErr[~inCalib.coeffValid] = np.nan 

661 

662 outCalib.coeffNum = inCalib.coeffNum 

663 outCalib.coeffValid = inCalib.coeffValid 

664 

665 return outCalib 

666 

667 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False): 

668 """Utility function to examine the final CT ratio set. 

669 

670 Parameters 

671 ---------- 

672 stepname : `str` 

673 State of processing to view. 

674 ratios : `dict` of `dict` of `np.ndarray` 

675 Array of measured CT ratios, indexed by source/victim 

676 amplifier. 

677 i : `str` 

678 Index of the source amplifier. 

679 j : `str` 

680 Index of the target amplifier. 

681 coeff : `float`, optional 

682 Coefficient calculated to plot along with the simple mean. 

683 valid : `bool`, optional 

684 Validity to be added to the plot title. 

685 """ 

686 frame = getDebugFrame(self._display, stepname) 

687 if frame: 

688 if i == j or ratios is None or len(ratios) < 1: 

689 pass 

690 

691 ratioList = ratios[i][j] 

692 if ratioList is None or len(ratioList) < 1: 

693 pass 

694 

695 mean = np.mean(ratioList) 

696 std = np.std(ratioList) 

697 import matplotlib.pyplot as plt 

698 figure = plt.figure(1) 

699 figure.clear() 

700 plt.hist(x=ratioList, bins=len(ratioList), 

701 cumulative=True, color='b', density=True, histtype='step') 

702 plt.xlabel("Measured pixel ratio") 

703 plt.ylabel(f"CDF: n={len(ratioList)}") 

704 plt.xlim(np.percentile(ratioList, [1.0, 99])) 

705 plt.axvline(x=mean, color="k") 

706 plt.axvline(x=coeff, color='g') 

707 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r') 

708 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r') 

709 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}") 

710 figure.show() 

711 

712 prompt = "Press Enter to continue: " 

713 while True: 

714 ans = input(prompt).lower() 

715 if ans in ("", "c",): 

716 break 

717 elif ans in ("pdb", "p",): 

718 import pdb 

719 pdb.set_trace() 

720 plt.close() 

721 

722 

723class MeasureCrosstalkConfig(Config): 

724 extract = ConfigurableField( 

725 target=CrosstalkExtractTask, 

726 doc="Task to measure pixel ratios.", 

727 ) 

728 solver = ConfigurableField( 

729 target=CrosstalkSolveTask, 

730 doc="Task to convert ratio lists to crosstalk coefficients.", 

731 ) 

732 

733 

734class MeasureCrosstalkTask(pipeBase.CmdLineTask): 

735 """Measure intra-detector crosstalk. 

736 

737 See also 

738 -------- 

739 lsst.ip.isr.crosstalk.CrosstalkCalib 

740 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask 

741 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask 

742 

743 Notes 

744 ----- 

745 The crosstalk this method measures assumes that when a bright 

746 pixel is found in one detector amplifier, all other detector 

747 amplifiers may see a signal change in the same pixel location 

748 (relative to the readout amplifier) as these other pixels are read 

749 out at the same time. 

750 

751 After processing each input exposure through a limited set of ISR 

752 stages, bright unmasked pixels above the threshold are identified. 

753 The potential CT signal is found by taking the ratio of the 

754 appropriate background-subtracted pixel value on the other 

755 amplifiers to the input value on the source amplifier. If the 

756 source amplifier has a large number of bright pixels as well, the 

757 background level may be elevated, leading to poor ratio 

758 measurements. 

759 

760 The set of ratios found between each pair of amplifiers across all 

761 input exposures is then gathered to produce the final CT 

762 coefficients. The sigma-clipped mean and sigma are returned from 

763 these sets of ratios, with the coefficient to supply to the ISR 

764 CrosstalkTask() being the multiplicative inverse of these values. 

765 

766 This Task simply calls the pipetask versions of the measure 

767 crosstalk code. 

768 """ 

769 ConfigClass = MeasureCrosstalkConfig 

770 _DefaultName = "measureCrosstalk" 

771 

772 # Let's use this instead of messing with parseAndRun. 

773 RunnerClass = DataRefListRunner 

774 

775 def __init__(self, **kwargs): 

776 super().__init__(**kwargs) 

777 self.makeSubtask("extract") 

778 self.makeSubtask("solver") 

779 

780 def runDataRef(self, dataRefList): 

781 """Run extract task on each of inputs in the dataRef list, then pass 

782 that to the solver task. 

783 

784 Parameters 

785 ---------- 

786 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

787 Data references for exposures for detectors to process. 

788 

789 Returns 

790 ------- 

791 results : `lsst.pipe.base.Struct` 

792 The results struct containing: 

793 

794 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib` 

795 Final crosstalk calibration. 

796 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

797 Provenance data for the new calibration. 

798 

799 Raises 

800 ------ 

801 RuntimeError 

802 Raised if multiple target detectors are supplied. 

803 """ 

804 dataRef = dataRefList[0] 

805 camera = dataRef.get("camera") 

806 

807 ratios = [] 

808 activeChip = None 

809 for dataRef in dataRefList: 

810 exposure = dataRef.get("postISRCCD") 

811 if activeChip: 

812 if exposure.getDetector().getName() != activeChip: 

813 raise RuntimeError("Too many input detectors supplied!") 

814 else: 

815 activeChip = exposure.getDetector().getName() 

816 

817 self.extract.debugView("extract", exposure) 

818 result = self.extract.run(exposure) 

819 ratios.append(result.outputRatios) 

820 

821 finalResults = self.solver.run(ratios, camera=camera) 

822 dataRef.put(finalResults.outputCrosstalk, "crosstalk") 

823 

824 return finalResults