Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import itertools 

22import numpy as np 

23from scipy.stats import norm 

24 

25from collections import defaultdict 

26 

27import lsst.pipe.base as pipeBase 

28import lsst.pipe.base.connectionTypes as cT 

29 

30from lsstDebug import getDebugFrame 

31from lsst.afw.detection import FootprintSet, Threshold 

32from lsst.afw.display import getDisplay 

33from lsst.pex.config import Config, Field, ListField, ConfigurableField 

34from lsst.ip.isr import CrosstalkCalib, IsrProvenance 

35from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

36 

37__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask", 

38 "CrosstalkSolveTask", "CrosstalkSolveConfig", 

39 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"] 

40 

41 

42class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "exposure", "detector")): 

44 inputExp = cT.Input( 

45 name="crosstalkInputs", 

46 doc="Input post-ISR processed exposure to measure crosstalk from.", 

47 storageClass="Exposure", 

48 dimensions=("instrument", "exposure", "detector"), 

49 multiple=False, 

50 ) 

51 # TODO: Depends on DM-21904. 

52 sourceExp = cT.Input( 

53 name="crosstalkSource", 

54 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.", 

55 storageClass="Exposure", 

56 dimensions=("instrument", "exposure", "detector"), 

57 multiple=True, 

58 deferLoad=True, 

59 # lookupFunction=None, 

60 ) 

61 

62 outputRatios = cT.Output( 

63 name="crosstalkRatios", 

64 doc="Extracted crosstalk pixel ratios.", 

65 storageClass="StructuredDataDict", 

66 dimensions=("instrument", "exposure", "detector"), 

67 ) 

68 outputFluxes = cT.Output( 

69 name="crosstalkFluxes", 

70 doc="Source pixel fluxes used in ratios.", 

71 storageClass="StructuredDataDict", 

72 dimensions=("instrument", "exposure", "detector"), 

73 ) 

74 

75 def __init__(self, *, config=None): 

76 super().__init__(config=config) 

77 # Discard sourceExp until DM-21904 allows full interchip 

78 # measurements. 

79 self.inputs.discard("sourceExp") 

80 

81 

82class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig, 

83 pipelineConnections=CrosstalkExtractConnections): 

84 """Configuration for the measurement of pixel ratios. 

85 """ 

86 doMeasureInterchip = Field( 

87 dtype=bool, 

88 default=False, 

89 doc="Measure inter-chip crosstalk as well?", 

90 ) 

91 threshold = Field( 

92 dtype=float, 

93 default=30000, 

94 doc="Minimum level of source pixels for which to measure crosstalk." 

95 ) 

96 ignoreSaturatedPixels = Field( 

97 dtype=bool, 

98 default=True, 

99 doc="Should saturated pixels be ignored?" 

100 ) 

101 badMask = ListField( 

102 dtype=str, 

103 default=["BAD", "INTRP"], 

104 doc="Mask planes to ignore when identifying source pixels." 

105 ) 

106 isTrimmed = Field( 

107 dtype=bool, 

108 default=True, 

109 doc="Is the input exposure trimmed?" 

110 ) 

111 

112 def validate(self): 

113 super().validate() 

114 

115 # Ensure the handling of the SAT mask plane is consistent 

116 # with the ignoreSaturatedPixels value. 

117 if self.ignoreSaturatedPixels: 

118 if 'SAT' not in self.badMask: 

119 self.badMask.append('SAT') 

120 else: 

121 if 'SAT' in self.badMask: 

122 self.badMask = [mask for mask in self.badMask if mask != 'SAT'] 

123 

124 

125class CrosstalkExtractTask(pipeBase.PipelineTask, 

126 pipeBase.CmdLineTask): 

127 """Task to measure pixel ratios to find crosstalk. 

128 """ 

129 ConfigClass = CrosstalkExtractConfig 

130 _DefaultName = 'cpCrosstalkExtract' 

131 

132 def run(self, inputExp, sourceExps=[]): 

133 """Measure pixel ratios between amplifiers in inputExp. 

134 

135 Extract crosstalk ratios between different amplifiers. 

136 

137 For pixels above ``config.threshold``, we calculate the ratio 

138 between each background-subtracted target amp and the source 

139 amp. We return a list of ratios for each pixel for each 

140 target/source combination, as nested dictionary containing the 

141 ratio. 

142 

143 Parameters 

144 ---------- 

145 inputExp : `lsst.afw.image.Exposure` 

146 Input exposure to measure pixel ratios on. 

147 sourceExp : `list` [`lsst.afw.image.Exposure`], optional 

148 List of chips to use as sources to measure inter-chip 

149 crosstalk. 

150 

151 Returns 

152 ------- 

153 results : `lsst.pipe.base.Struct` 

154 The results struct containing: 

155 

156 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]] 

157 A catalog of ratio lists. The dictionaries are 

158 indexed such that: 

159 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp] 

160 contains the ratio list for that combination. 

161 ``outputFluxes`` : `dict` [`dict` [`list`]] 

162 A catalog of flux lists. The dictionaries are 

163 indexed such that: 

164 outputFluxes[sourceChip][sourceAmp] 

165 contains the flux list used in the outputRatios. 

166 

167 Notes 

168 ----- 

169 The lsstDebug.Info() method can be rewritten for __name__ = 

170 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters: 

171 

172 debug.display['extract'] : `bool` 

173 Display the exposure under consideration, with the pixels used 

174 for crosstalk measurement indicated by the DETECTED mask plane. 

175 debug.display['pixels'] : `bool` 

176 Display a plot of the ratio calculated for each pixel used in this 

177 exposure, split by amplifier pairs. The median value is listed 

178 for reference. 

179 """ 

180 outputRatios = defaultdict(lambda: defaultdict(dict)) 

181 outputFluxes = defaultdict(lambda: defaultdict(dict)) 

182 

183 threshold = self.config.threshold 

184 badPixels = list(self.config.badMask) 

185 

186 targetDetector = inputExp.getDetector() 

187 targetChip = targetDetector.getName() 

188 

189 # Always look at the target chip first, then go to any other supplied exposures. 

190 sourceExtractExps = [inputExp] 

191 sourceExtractExps.extend(sourceExps) 

192 

193 self.log.info("Measuring full detector background for target: %s", targetChip) 

194 targetIm = inputExp.getMaskedImage() 

195 FootprintSet(targetIm, Threshold(threshold), "DETECTED") 

196 detected = targetIm.getMask().getPlaneBitMask("DETECTED") 

197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) 

198 

199 self.debugView('extract', inputExp) 

200 

201 for sourceExp in sourceExtractExps: 

202 sourceDetector = sourceExp.getDetector() 

203 sourceChip = sourceDetector.getName() 

204 sourceIm = sourceExp.getMaskedImage() 

205 bad = sourceIm.getMask().getPlaneBitMask(badPixels) 

206 self.log.info("Measuring crosstalk from source: %s", sourceChip) 

207 

208 if sourceExp != inputExp: 

209 FootprintSet(sourceIm, Threshold(threshold), "DETECTED") 

210 detected = sourceIm.getMask().getPlaneBitMask("DETECTED") 

211 

212 # The dictionary of amp-to-amp ratios for this pair of source->target detectors. 

213 ratioDict = defaultdict(lambda: defaultdict(list)) 

214 extractedCount = 0 

215 

216 for sourceAmp in sourceDetector: 

217 sourceAmpName = sourceAmp.getName() 

218 sourceAmpImage = sourceIm[sourceAmp.getBBox()] 

219 sourceMask = sourceAmpImage.mask.array 

220 select = ((sourceMask & detected > 0) & 

221 (sourceMask & bad == 0) & 

222 np.isfinite(sourceAmpImage.image.array)) 

223 count = np.sum(select) 

224 self.log.debug(" Source amplifier: %s", sourceAmpName) 

225 

226 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select] 

227 

228 for targetAmp in targetDetector: 

229 # iterate over targetExposure 

230 targetAmpName = targetAmp.getName() 

231 if sourceAmpName == targetAmpName and sourceChip == targetChip: 

232 ratioDict[sourceAmpName][targetAmpName] = [] 

233 continue 

234 self.log.debug(" Target amplifier: %s", targetAmpName) 

235 

236 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image, 

237 targetAmp, sourceAmp, 

238 isTrimmed=self.config.isTrimmed) 

239 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select] 

240 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist() 

241 extractedCount += count 

242 

243 self.debugPixels('pixels', 

244 sourceAmpImage.image.array[select], 

245 targetAmpImage.array[select] - bg, 

246 sourceAmpName, targetAmpName) 

247 

248 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)", 

249 extractedCount, sourceChip, targetChip, bg) 

250 outputRatios[targetChip][sourceChip] = ratioDict 

251 

252 return pipeBase.Struct( 

253 outputRatios=outputRatios, 

254 outputFluxes=outputFluxes 

255 ) 

256 

257 def debugView(self, stepname, exposure): 

258 """Utility function to examine the image being processed. 

259 

260 Parameters 

261 ---------- 

262 stepname : `str` 

263 State of processing to view. 

264 exposure : `lsst.afw.image.Exposure` 

265 Exposure to view. 

266 """ 

267 frame = getDebugFrame(self._display, stepname) 

268 if frame: 

269 display = getDisplay(frame) 

270 display.scale('asinh', 'zscale') 

271 display.mtv(exposure) 

272 

273 prompt = "Press Enter to continue: " 

274 while True: 

275 ans = input(prompt).lower() 

276 if ans in ("", "c",): 

277 break 

278 

279 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName): 

280 """Utility function to examine the CT ratio pixel values. 

281 

282 Parameters 

283 ---------- 

284 stepname : `str` 

285 State of processing to view. 

286 pixelsIn : `np.ndarray` 

287 Pixel values from the potential crosstalk source. 

288 pixelsOut : `np.ndarray` 

289 Pixel values from the potential crosstalk target. 

290 sourceName : `str` 

291 Source amplifier name 

292 targetName : `str` 

293 Target amplifier name 

294 """ 

295 frame = getDebugFrame(self._display, stepname) 

296 if frame: 

297 import matplotlib.pyplot as plt 

298 figure = plt.figure(1) 

299 figure.clear() 

300 

301 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

302 axes.plt(pixelsIn, pixelsOut / pixelsIn, 'k+') 

303 plt.xlabel("Source amplifier pixel value") 

304 plt.ylabel("Measured pixel ratio") 

305 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: " 

306 f"{(np.median(pixelsOut / pixelsIn))}") 

307 figure.show() 

308 

309 prompt = "Press Enter to continue: " 

310 while True: 

311 ans = input(prompt).lower() 

312 if ans in ("", "c",): 

313 break 

314 plt.close() 

315 

316 

317class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections, 

318 dimensions=("instrument", "detector")): 

319 inputRatios = cT.Input( 

320 name="crosstalkRatios", 

321 doc="Ratios measured for an input exposure.", 

322 storageClass="StructuredDataDict", 

323 dimensions=("instrument", "exposure", "detector"), 

324 multiple=True, 

325 ) 

326 inputFluxes = cT.Input( 

327 name="crosstalkFluxes", 

328 doc="Fluxes of CT source pixels, for nonlinear fits.", 

329 storageClass="StructuredDataDict", 

330 dimensions=("instrument", "exposure", "detector"), 

331 multiple=True, 

332 ) 

333 camera = cT.PrerequisiteInput( 

334 name="camera", 

335 doc="Camera the input data comes from.", 

336 storageClass="Camera", 

337 dimensions=("instrument", "calibration_label"), 

338 ) 

339 

340 outputCrosstalk = cT.Output( 

341 name="crosstalkProposal", 

342 doc="Output proposed crosstalk calibration.", 

343 storageClass="CrosstalkCalib", 

344 dimensions=("instrument", "detector"), 

345 multiple=False, 

346 ) 

347 

348 def __init__(self, *, config=None): 

349 super().__init__(config=config) 

350 

351 if config.fluxOrder == 0: 

352 self.inputs.discard("inputFluxes") 

353 

354 

355class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig, 

356 pipelineConnections=CrosstalkSolveConnections): 

357 """Configuration for the solving of crosstalk from pixel ratios. 

358 """ 

359 rejIter = Field( 

360 dtype=int, 

361 default=3, 

362 doc="Number of rejection iterations for final coefficient calculation.", 

363 ) 

364 rejSigma = Field( 

365 dtype=float, 

366 default=2.0, 

367 doc="Rejection threshold (sigma) for final coefficient calculation.", 

368 ) 

369 fluxOrder = Field( 

370 dtype=int, 

371 default=0, 

372 doc="Polynomial order in source flux to fit crosstalk.", 

373 ) 

374 doFiltering = Field( 

375 dtype=bool, 

376 default=False, 

377 doc="Filter generated crosstalk to remove marginal measurements.", 

378 ) 

379 

380 

381class CrosstalkSolveTask(pipeBase.PipelineTask, 

382 pipeBase.CmdLineTask): 

383 """Task to solve crosstalk from pixel ratios. 

384 """ 

385 ConfigClass = CrosstalkSolveConfig 

386 _DefaultName = 'cpCrosstalkSolve' 

387 

388 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

389 """Ensure that the input and output dimensions are passed along. 

390 

391 Parameters 

392 ---------- 

393 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

394 Butler to operate on. 

395 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

396 Input data refs to load. 

397 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

398 Output data refs to persist. 

399 """ 

400 inputs = butlerQC.get(inputRefs) 

401 

402 # Use the dimensions to set calib/provenance information. 

403 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios] 

404 inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName() 

405 

406 outputs = self.run(**inputs) 

407 butlerQC.put(outputs, outputRefs) 

408 

409 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None): 

410 """Combine ratios to produce crosstalk coefficients. 

411 

412 Parameters 

413 ---------- 

414 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]] 

415 A list of nested dictionaries of ratios indexed by target 

416 and source chip, then by target and source amplifier. 

417 inputFluxes : `list` [`dict` [`dict` [`list`]]] 

418 A list of nested dictionaries of source pixel fluxes, indexed 

419 by source chip and amplifier. 

420 camera : `lsst.afw.cameraGeom.Camera` 

421 Input camera. 

422 inputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

423 DataIds to use to construct provenance. 

424 outputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

425 DataIds to use to populate the output calibration. 

426 

427 Returns 

428 ------- 

429 results : `lsst.pipe.base.Struct` 

430 The results struct containing: 

431 

432 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib` 

433 Final crosstalk calibration. 

434 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

435 Provenance data for the new calibration. 

436 

437 Raises 

438 ------ 

439 RuntimeError 

440 Raised if the input data contains multiple target detectors. 

441 

442 Notes 

443 ----- 

444 The lsstDebug.Info() method can be rewritten for __name__ = 

445 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

446 

447 debug.display['reduce'] : `bool` 

448 Display a histogram of the combined ratio measurements for 

449 a pair of source/target amplifiers from all input 

450 exposures/detectors. 

451 

452 """ 

453 if outputDims: 

454 calibChip = outputDims['detector'] 

455 instrument = outputDims['instrument'] 

456 else: 

457 # calibChip needs to be set manually in Gen2. 

458 calibChip = None 

459 instrument = None 

460 

461 self.log.info("Combining measurements from %d ratios and %d fluxes", 

462 len(inputRatios), len(inputFluxes) if inputFluxes else 0) 

463 

464 if inputFluxes is None: 

465 inputFluxes = [None for exp in inputRatios] 

466 

467 combinedRatios = defaultdict(lambda: defaultdict(list)) 

468 combinedFluxes = defaultdict(lambda: defaultdict(list)) 

469 for ratioDict, fluxDict in zip(inputRatios, inputFluxes): 

470 for targetChip in ratioDict: 

471 if calibChip and targetChip != calibChip: 

472 raise RuntimeError("Received multiple target chips!") 

473 

474 sourceChip = targetChip 

475 if sourceChip in ratioDict[targetChip]: 

476 ratios = ratioDict[targetChip][sourceChip] 

477 

478 for targetAmp in ratios: 

479 for sourceAmp in ratios[targetAmp]: 

480 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp]) 

481 if fluxDict: 

482 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp]) 

483 # TODO: DM-21904 

484 # Iterating over all other entries in ratioDict[targetChip] will yield 

485 # inter-chip terms. 

486 

487 for targetAmp in combinedRatios: 

488 for sourceAmp in combinedRatios[targetAmp]: 

489 self.log.info("Read %d pixels for %s -> %s", 

490 len(combinedRatios[targetAmp][sourceAmp]), 

491 targetAmp, sourceAmp) 

492 if len(combinedRatios[targetAmp][sourceAmp]) > 1: 

493 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp) 

494 

495 if self.config.fluxOrder == 0: 

496 self.log.info("Fitting crosstalk coefficients.") 

497 calib = self.measureCrosstalkCoefficients(combinedRatios, 

498 self.config.rejIter, self.config.rejSigma) 

499 else: 

500 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.") 

501 

502 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid)) 

503 

504 if self.config.doFiltering: 

505 # This step will apply the calculated validity values to 

506 # censor poorly measured coefficients. 

507 self.log.info("Filtering measured crosstalk to remove invalid solutions.") 

508 calib = self.filterCrosstalkCalib(calib) 

509 

510 # Populate the remainder of the calibration information. 

511 calib.hasCrosstalk = True 

512 calib.interChip = {} 

513 calib._detectorName = calibChip 

514 if camera: 

515 for chip in camera: 

516 if chip.getName() == calibChip: 

517 calib._detectorSerial = chip.getSerial() 

518 calib._instrument = instrument 

519 calib.updateMetadata() 

520 

521 # Make an IsrProvenance(). 

522 provenance = IsrProvenance(calibType="CROSSTALK") 

523 provenance._detectorName = calibChip 

524 if inputDims: 

525 provenance.fromDataIds(inputDims) 

526 provenance._instrument = instrument 

527 provenance.updateMetadata() 

528 

529 return pipeBase.Struct( 

530 outputCrosstalk=calib, 

531 outputProvenance=provenance, 

532 ) 

533 

534 def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma): 

535 """Measure crosstalk coefficients from the ratios. 

536 

537 Given a list of ratios for each target/source amp combination, 

538 we measure a sigma clipped mean and error. 

539 

540 The coefficient errors returned are the standard deviation of 

541 the final set of clipped input ratios. 

542 

543 Parameters 

544 ---------- 

545 ratios : `dict` of `dict` of `numpy.ndarray` 

546 Catalog of arrays of ratios. 

547 rejIter : `int` 

548 Number of rejection iterations. 

549 rejSigma : `float` 

550 Rejection threshold (sigma). 

551 

552 Returns 

553 ------- 

554 calib : `lsst.ip.isr.CrosstalkCalib` 

555 The output crosstalk calibration. 

556 

557 Notes 

558 ----- 

559 The lsstDebug.Info() method can be rewritten for __name__ = 

560 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

561 

562 debug.display['measure'] : `bool` 

563 Display the CDF of the combined ratio measurements for 

564 a pair of source/target amplifiers from the final set of 

565 clipped input ratios. 

566 """ 

567 calib = CrosstalkCalib(nAmp=len(ratios)) 

568 

569 # Calibration stores coefficients as a numpy ndarray. 

570 ordering = list(ratios.keys()) 

571 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)): 

572 if ii == jj: 

573 values = [0.0] 

574 else: 

575 values = np.array(ratios[ordering[ii]][ordering[jj]]) 

576 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

577 

578 calib.coeffNum[ii][jj] = len(values) 

579 

580 if len(values) == 0: 

581 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

582 calib.coeffs[ii][jj] = np.nan 

583 calib.coeffErr[ii][jj] = np.nan 

584 calib.coeffValid[ii][jj] = False 

585 else: 

586 if ii != jj: 

587 for rej in range(rejIter): 

588 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

589 sigma = 0.741*(hi - lo) 

590 good = np.abs(values - med) < rejSigma*sigma 

591 if good.sum() == len(good): 

592 break 

593 values = values[good] 

594 

595 calib.coeffs[ii][jj] = np.mean(values) 

596 if calib.coeffNum[ii][jj] == 1: 

597 calib.coeffErr[ii][jj] = np.nan 

598 else: 

599 correctionFactor = self.sigmaClipCorrection(rejSigma) 

600 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor 

601 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) > 

602 calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj])) 

603 

604 if calib.coeffNum[ii][jj] > 1: 

605 self.debugRatios('measure', ratios, ordering[ii], ordering[jj], 

606 calib.coeffs[ii][jj], calib.coeffValid[ii][jj]) 

607 

608 return calib 

609 

610 @staticmethod 

611 def sigmaClipCorrection(nSigClip): 

612 """Correct measured sigma to account for clipping. 

613 

614 If we clip our input data and then measure sigma, then the 

615 measured sigma is smaller than the true value because real 

616 points beyond the clip threshold have been removed. This is a 

617 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the 

618 default parameters for measure crosstalk use nSigClip=2.0. 

619 This causes the measured sigma to be about 15% smaller than 

620 real. This formula corrects the issue, for the symmetric case 

621 (upper clip threshold equal to lower clip threshold). 

622 

623 Parameters 

624 ---------- 

625 nSigClip : `float` 

626 Number of sigma the measurement was clipped by. 

627 

628 Returns 

629 ------- 

630 scaleFactor : `float` 

631 Scale factor to increase the measured sigma by. 

632 

633 """ 

634 varFactor = 1.0 + (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip)) 

635 return 1.0 / np.sqrt(varFactor) 

636 

637 @staticmethod 

638 def filterCrosstalkCalib(inCalib): 

639 """Apply valid constraints to the measured values. 

640 

641 Any measured coefficient that is determined to be invalid is 

642 set to zero, and has the error set to nan. The validation is 

643 determined by checking that the measured coefficient is larger 

644 than the calculated standard error of the mean. 

645 

646 Parameters 

647 ---------- 

648 inCalib : `lsst.ip.isr.CrosstalkCalib` 

649 Input calibration to filter. 

650 

651 Returns 

652 ------- 

653 outCalib : `lsst.ip.isr.CrosstalkCalib` 

654 Filtered calibration. 

655 """ 

656 outCalib = CrosstalkCalib() 

657 outCalib.numAmps = inCalib.numAmps 

658 

659 outCalib.coeffs = inCalib.coeffs 

660 outCalib.coeffs[~inCalib.coeffValid] = 0.0 

661 

662 outCalib.coeffErr = inCalib.coeffErr 

663 outCalib.coeffErr[~inCalib.coeffValid] = np.nan 

664 

665 outCalib.coeffNum = inCalib.coeffNum 

666 outCalib.coeffValid = inCalib.coeffValid 

667 

668 return outCalib 

669 

670 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False): 

671 """Utility function to examine the final CT ratio set. 

672 

673 Parameters 

674 ---------- 

675 stepname : `str` 

676 State of processing to view. 

677 ratios : `dict` of `dict` of `np.ndarray` 

678 Array of measured CT ratios, indexed by source/victim 

679 amplifier. 

680 i : `str` 

681 Index of the source amplifier. 

682 j : `str` 

683 Index of the target amplifier. 

684 coeff : `float`, optional 

685 Coefficient calculated to plot along with the simple mean. 

686 valid : `bool`, optional 

687 Validity to be added to the plot title. 

688 """ 

689 frame = getDebugFrame(self._display, stepname) 

690 if frame: 

691 if i == j or ratios is None or len(ratios) < 1: 

692 pass 

693 

694 ratioList = ratios[i][j] 

695 if ratioList is None or len(ratioList) < 1: 

696 pass 

697 

698 mean = np.mean(ratioList) 

699 std = np.std(ratioList) 

700 import matplotlib.pyplot as plt 

701 figure = plt.figure(1) 

702 figure.clear() 

703 plt.hist(x=ratioList, bins=len(ratioList), 

704 cumulative=True, color='b', density=True, histtype='step') 

705 plt.xlabel("Measured pixel ratio") 

706 plt.ylabel(f"CDF: n={len(ratioList)}") 

707 plt.xlim(np.percentile(ratioList, [1.0, 99])) 

708 plt.axvline(x=mean, color="k") 

709 plt.axvline(x=coeff, color='g') 

710 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r') 

711 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r') 

712 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}") 

713 figure.show() 

714 

715 prompt = "Press Enter to continue: " 

716 while True: 

717 ans = input(prompt).lower() 

718 if ans in ("", "c",): 

719 break 

720 elif ans in ("pdb", "p",): 

721 import pdb 

722 pdb.set_trace() 

723 plt.close() 

724 

725 

726class MeasureCrosstalkConfig(Config): 

727 extract = ConfigurableField( 

728 target=CrosstalkExtractTask, 

729 doc="Task to measure pixel ratios.", 

730 ) 

731 solver = ConfigurableField( 

732 target=CrosstalkSolveTask, 

733 doc="Task to convert ratio lists to crosstalk coefficients.", 

734 ) 

735 

736 

737class MeasureCrosstalkTask(pipeBase.CmdLineTask): 

738 """Measure intra-detector crosstalk. 

739 

740 See also 

741 -------- 

742 lsst.ip.isr.crosstalk.CrosstalkCalib 

743 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask 

744 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask 

745 

746 Notes 

747 ----- 

748 The crosstalk this method measures assumes that when a bright 

749 pixel is found in one detector amplifier, all other detector 

750 amplifiers may see a signal change in the same pixel location 

751 (relative to the readout amplifier) as these other pixels are read 

752 out at the same time. 

753 

754 After processing each input exposure through a limited set of ISR 

755 stages, bright unmasked pixels above the threshold are identified. 

756 The potential CT signal is found by taking the ratio of the 

757 appropriate background-subtracted pixel value on the other 

758 amplifiers to the input value on the source amplifier. If the 

759 source amplifier has a large number of bright pixels as well, the 

760 background level may be elevated, leading to poor ratio 

761 measurements. 

762 

763 The set of ratios found between each pair of amplifiers across all 

764 input exposures is then gathered to produce the final CT 

765 coefficients. The sigma-clipped mean and sigma are returned from 

766 these sets of ratios, with the coefficient to supply to the ISR 

767 CrosstalkTask() being the multiplicative inverse of these values. 

768 

769 This Task simply calls the pipetask versions of the measure 

770 crosstalk code. 

771 """ 

772 ConfigClass = MeasureCrosstalkConfig 

773 _DefaultName = "measureCrosstalk" 

774 

775 # Let's use this instead of messing with parseAndRun. 

776 RunnerClass = DataRefListRunner 

777 

778 def __init__(self, **kwargs): 

779 super().__init__(**kwargs) 

780 self.makeSubtask("extract") 

781 self.makeSubtask("solver") 

782 

783 def runDataRef(self, dataRefList): 

784 """Run extract task on each of inputs in the dataRef list, then pass 

785 that to the solver task. 

786 

787 Parameters 

788 ---------- 

789 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

790 Data references for exposures for detectors to process. 

791 

792 Returns 

793 ------- 

794 results : `lsst.pipe.base.Struct` 

795 The results struct containing: 

796 

797 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib` 

798 Final crosstalk calibration. 

799 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

800 Provenance data for the new calibration. 

801 

802 Raises 

803 ------ 

804 RuntimeError 

805 Raised if multiple target detectors are supplied. 

806 """ 

807 dataRef = dataRefList[0] 

808 camera = dataRef.get("camera") 

809 

810 ratios = [] 

811 activeChip = None 

812 for dataRef in dataRefList: 

813 exposure = dataRef.get("postISRCCD") 

814 if activeChip: 

815 if exposure.getDetector().getName() != activeChip: 

816 raise RuntimeError("Too many input detectors supplied!") 

817 else: 

818 activeChip = exposure.getDetector().getName() 

819 

820 self.extract.debugView("extract", exposure) 

821 result = self.extract.run(exposure) 

822 ratios.append(result.outputRatios) 

823 

824 finalResults = self.solver.run(ratios, camera=camera) 

825 dataRef.put(finalResults.outputCrosstalk, "crosstalk") 

826 

827 return finalResults