Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import itertools 

22import numpy as np 

23 

24from collections import defaultdict 

25 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28 

29from lsstDebug import getDebugFrame 

30from lsst.afw.detection import FootprintSet, Threshold 

31from lsst.afw.display import getDisplay 

32from lsst.pex.config import Config, Field, ListField, ConfigurableField 

33from lsst.ip.isr import CrosstalkCalib, IsrProvenance 

34from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

35from lsst.cp.pipe.utils import (ddict2dict, sigmaClipCorrection) 

36 

37from ._lookupStaticCalibration import lookupStaticCalibration 

38 

39__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask", 

40 "CrosstalkSolveTask", "CrosstalkSolveConfig", 

41 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"] 

42 

43 

44class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections, 

45 dimensions=("instrument", "exposure", "detector")): 

46 inputExp = cT.Input( 

47 name="crosstalkInputs", 

48 doc="Input post-ISR processed exposure to measure crosstalk from.", 

49 storageClass="Exposure", 

50 dimensions=("instrument", "exposure", "detector"), 

51 multiple=False, 

52 ) 

53 # TODO: Depends on DM-21904. 

54 sourceExp = cT.Input( 

55 name="crosstalkSource", 

56 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.", 

57 storageClass="Exposure", 

58 dimensions=("instrument", "exposure", "detector"), 

59 multiple=True, 

60 deferLoad=True, 

61 # lookupFunction=None, 

62 ) 

63 

64 outputRatios = cT.Output( 

65 name="crosstalkRatios", 

66 doc="Extracted crosstalk pixel ratios.", 

67 storageClass="StructuredDataDict", 

68 dimensions=("instrument", "exposure", "detector"), 

69 ) 

70 outputFluxes = cT.Output( 

71 name="crosstalkFluxes", 

72 doc="Source pixel fluxes used in ratios.", 

73 storageClass="StructuredDataDict", 

74 dimensions=("instrument", "exposure", "detector"), 

75 ) 

76 

77 def __init__(self, *, config=None): 

78 super().__init__(config=config) 

79 # Discard sourceExp until DM-21904 allows full interchip 

80 # measurements. 

81 self.inputs.discard("sourceExp") 

82 

83 

84class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=CrosstalkExtractConnections): 

86 """Configuration for the measurement of pixel ratios. 

87 """ 

88 doMeasureInterchip = Field( 

89 dtype=bool, 

90 default=False, 

91 doc="Measure inter-chip crosstalk as well?", 

92 ) 

93 threshold = Field( 

94 dtype=float, 

95 default=30000, 

96 doc="Minimum level of source pixels for which to measure crosstalk." 

97 ) 

98 ignoreSaturatedPixels = Field( 

99 dtype=bool, 

100 default=True, 

101 doc="Should saturated pixels be ignored?" 

102 ) 

103 badMask = ListField( 

104 dtype=str, 

105 default=["BAD", "INTRP"], 

106 doc="Mask planes to ignore when identifying source pixels." 

107 ) 

108 isTrimmed = Field( 

109 dtype=bool, 

110 default=True, 

111 doc="Is the input exposure trimmed?" 

112 ) 

113 

114 def validate(self): 

115 super().validate() 

116 

117 # Ensure the handling of the SAT mask plane is consistent 

118 # with the ignoreSaturatedPixels value. 

119 if self.ignoreSaturatedPixels: 

120 if 'SAT' not in self.badMask: 

121 self.badMask.append('SAT') 

122 else: 

123 if 'SAT' in self.badMask: 

124 self.badMask = [mask for mask in self.badMask if mask != 'SAT'] 

125 

126 

127class CrosstalkExtractTask(pipeBase.PipelineTask, 

128 pipeBase.CmdLineTask): 

129 """Task to measure pixel ratios to find crosstalk. 

130 """ 

131 ConfigClass = CrosstalkExtractConfig 

132 _DefaultName = 'cpCrosstalkExtract' 

133 

134 def run(self, inputExp, sourceExps=[]): 

135 """Measure pixel ratios between amplifiers in inputExp. 

136 

137 Extract crosstalk ratios between different amplifiers. 

138 

139 For pixels above ``config.threshold``, we calculate the ratio 

140 between each background-subtracted target amp and the source 

141 amp. We return a list of ratios for each pixel for each 

142 target/source combination, as nested dictionary containing the 

143 ratio. 

144 

145 Parameters 

146 ---------- 

147 inputExp : `lsst.afw.image.Exposure` 

148 Input exposure to measure pixel ratios on. 

149 sourceExp : `list` [`lsst.afw.image.Exposure`], optional 

150 List of chips to use as sources to measure inter-chip 

151 crosstalk. 

152 

153 Returns 

154 ------- 

155 results : `lsst.pipe.base.Struct` 

156 The results struct containing: 

157 

158 ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]] 

159 A catalog of ratio lists. The dictionaries are 

160 indexed such that: 

161 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp] 

162 contains the ratio list for that combination. 

163 ``outputFluxes`` : `dict` [`dict` [`list`]] 

164 A catalog of flux lists. The dictionaries are 

165 indexed such that: 

166 outputFluxes[sourceChip][sourceAmp] 

167 contains the flux list used in the outputRatios. 

168 

169 Notes 

170 ----- 

171 The lsstDebug.Info() method can be rewritten for __name__ = 

172 `lsst.cp.pipe.measureCrosstalk`, and supports the parameters: 

173 

174 debug.display['extract'] : `bool` 

175 Display the exposure under consideration, with the pixels used 

176 for crosstalk measurement indicated by the DETECTED mask plane. 

177 debug.display['pixels'] : `bool` 

178 Display a plot of the ratio calculated for each pixel used in this 

179 exposure, split by amplifier pairs. The median value is listed 

180 for reference. 

181 """ 

182 outputRatios = defaultdict(lambda: defaultdict(dict)) 

183 outputFluxes = defaultdict(lambda: defaultdict(dict)) 

184 

185 threshold = self.config.threshold 

186 badPixels = list(self.config.badMask) 

187 

188 targetDetector = inputExp.getDetector() 

189 targetChip = targetDetector.getName() 

190 

191 # Always look at the target chip first, then go to any other supplied exposures. 

192 sourceExtractExps = [inputExp] 

193 sourceExtractExps.extend(sourceExps) 

194 

195 self.log.info("Measuring full detector background for target: %s", targetChip) 

196 targetIm = inputExp.getMaskedImage() 

197 FootprintSet(targetIm, Threshold(threshold), "DETECTED") 

198 detected = targetIm.getMask().getPlaneBitMask("DETECTED") 

199 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) 

200 

201 self.debugView('extract', inputExp) 

202 

203 for sourceExp in sourceExtractExps: 

204 sourceDetector = sourceExp.getDetector() 

205 sourceChip = sourceDetector.getName() 

206 sourceIm = sourceExp.getMaskedImage() 

207 bad = sourceIm.getMask().getPlaneBitMask(badPixels) 

208 self.log.info("Measuring crosstalk from source: %s", sourceChip) 

209 

210 if sourceExp != inputExp: 

211 FootprintSet(sourceIm, Threshold(threshold), "DETECTED") 

212 detected = sourceIm.getMask().getPlaneBitMask("DETECTED") 

213 

214 # The dictionary of amp-to-amp ratios for this pair of source->target detectors. 

215 ratioDict = defaultdict(lambda: defaultdict(list)) 

216 extractedCount = 0 

217 

218 for sourceAmp in sourceDetector: 

219 sourceAmpName = sourceAmp.getName() 

220 sourceAmpImage = sourceIm[sourceAmp.getBBox()] 

221 sourceMask = sourceAmpImage.mask.array 

222 select = ((sourceMask & detected > 0) 

223 & (sourceMask & bad == 0) 

224 & np.isfinite(sourceAmpImage.image.array)) 

225 count = np.sum(select) 

226 self.log.debug(" Source amplifier: %s", sourceAmpName) 

227 

228 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist() 

229 

230 for targetAmp in targetDetector: 

231 # iterate over targetExposure 

232 targetAmpName = targetAmp.getName() 

233 if sourceAmpName == targetAmpName and sourceChip == targetChip: 

234 ratioDict[sourceAmpName][targetAmpName] = [] 

235 continue 

236 self.log.debug(" Target amplifier: %s", targetAmpName) 

237 

238 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image, 

239 targetAmp, sourceAmp, 

240 isTrimmed=self.config.isTrimmed) 

241 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select] 

242 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist() 

243 extractedCount += count 

244 

245 self.debugPixels('pixels', 

246 sourceAmpImage.image.array[select], 

247 targetAmpImage.array[select] - bg, 

248 sourceAmpName, targetAmpName) 

249 

250 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)", 

251 extractedCount, sourceChip, targetChip, bg) 

252 outputRatios[targetChip][sourceChip] = ratioDict 

253 

254 return pipeBase.Struct( 

255 outputRatios=ddict2dict(outputRatios), 

256 outputFluxes=ddict2dict(outputFluxes) 

257 ) 

258 

259 def debugView(self, stepname, exposure): 

260 """Utility function to examine the image being processed. 

261 

262 Parameters 

263 ---------- 

264 stepname : `str` 

265 State of processing to view. 

266 exposure : `lsst.afw.image.Exposure` 

267 Exposure to view. 

268 """ 

269 frame = getDebugFrame(self._display, stepname) 

270 if frame: 

271 display = getDisplay(frame) 

272 display.scale('asinh', 'zscale') 

273 display.mtv(exposure) 

274 

275 prompt = "Press Enter to continue: " 

276 while True: 

277 ans = input(prompt).lower() 

278 if ans in ("", "c",): 

279 break 

280 

281 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName): 

282 """Utility function to examine the CT ratio pixel values. 

283 

284 Parameters 

285 ---------- 

286 stepname : `str` 

287 State of processing to view. 

288 pixelsIn : `np.ndarray` 

289 Pixel values from the potential crosstalk source. 

290 pixelsOut : `np.ndarray` 

291 Pixel values from the potential crosstalk target. 

292 sourceName : `str` 

293 Source amplifier name 

294 targetName : `str` 

295 Target amplifier name 

296 """ 

297 frame = getDebugFrame(self._display, stepname) 

298 if frame: 

299 import matplotlib.pyplot as plt 

300 figure = plt.figure(1) 

301 figure.clear() 

302 

303 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

304 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+') 

305 plt.xlabel("Source amplifier pixel value") 

306 plt.ylabel("Measured pixel ratio") 

307 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: " 

308 f"{(np.median(pixelsOut / pixelsIn))}") 

309 figure.show() 

310 

311 prompt = "Press Enter to continue: " 

312 while True: 

313 ans = input(prompt).lower() 

314 if ans in ("", "c",): 

315 break 

316 plt.close() 

317 

318 

319class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections, 

320 dimensions=("instrument", "detector")): 

321 inputRatios = cT.Input( 

322 name="crosstalkRatios", 

323 doc="Ratios measured for an input exposure.", 

324 storageClass="StructuredDataDict", 

325 dimensions=("instrument", "exposure", "detector"), 

326 multiple=True, 

327 ) 

328 inputFluxes = cT.Input( 

329 name="crosstalkFluxes", 

330 doc="Fluxes of CT source pixels, for nonlinear fits.", 

331 storageClass="StructuredDataDict", 

332 dimensions=("instrument", "exposure", "detector"), 

333 multiple=True, 

334 ) 

335 camera = cT.PrerequisiteInput( 

336 name="camera", 

337 doc="Camera the input data comes from.", 

338 storageClass="Camera", 

339 dimensions=("instrument",), 

340 isCalibration=True, 

341 lookupFunction=lookupStaticCalibration, 

342 ) 

343 

344 outputCrosstalk = cT.Output( 

345 name="crosstalk", 

346 doc="Output proposed crosstalk calibration.", 

347 storageClass="CrosstalkCalib", 

348 dimensions=("instrument", "detector"), 

349 multiple=False, 

350 isCalibration=True, 

351 ) 

352 

353 def __init__(self, *, config=None): 

354 super().__init__(config=config) 

355 

356 if config.fluxOrder == 0: 

357 self.inputs.discard("inputFluxes") 

358 

359 

360class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig, 

361 pipelineConnections=CrosstalkSolveConnections): 

362 """Configuration for the solving of crosstalk from pixel ratios. 

363 """ 

364 rejIter = Field( 

365 dtype=int, 

366 default=3, 

367 doc="Number of rejection iterations for final coefficient calculation.", 

368 ) 

369 rejSigma = Field( 

370 dtype=float, 

371 default=2.0, 

372 doc="Rejection threshold (sigma) for final coefficient calculation.", 

373 ) 

374 fluxOrder = Field( 

375 dtype=int, 

376 default=0, 

377 doc="Polynomial order in source flux to fit crosstalk.", 

378 ) 

379 doFiltering = Field( 

380 dtype=bool, 

381 default=False, 

382 doc="Filter generated crosstalk to remove marginal measurements.", 

383 ) 

384 

385 

386class CrosstalkSolveTask(pipeBase.PipelineTask, 

387 pipeBase.CmdLineTask): 

388 """Task to solve crosstalk from pixel ratios. 

389 """ 

390 ConfigClass = CrosstalkSolveConfig 

391 _DefaultName = 'cpCrosstalkSolve' 

392 

393 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

394 """Ensure that the input and output dimensions are passed along. 

395 

396 Parameters 

397 ---------- 

398 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

399 Butler to operate on. 

400 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

401 Input data refs to load. 

402 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

403 Output data refs to persist. 

404 """ 

405 inputs = butlerQC.get(inputRefs) 

406 

407 # Use the dimensions to set calib/provenance information. 

408 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios] 

409 inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName() 

410 

411 outputs = self.run(**inputs) 

412 butlerQC.put(outputs, outputRefs) 

413 

414 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None): 

415 """Combine ratios to produce crosstalk coefficients. 

416 

417 Parameters 

418 ---------- 

419 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]] 

420 A list of nested dictionaries of ratios indexed by target 

421 and source chip, then by target and source amplifier. 

422 inputFluxes : `list` [`dict` [`dict` [`list`]]] 

423 A list of nested dictionaries of source pixel fluxes, indexed 

424 by source chip and amplifier. 

425 camera : `lsst.afw.cameraGeom.Camera` 

426 Input camera. 

427 inputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

428 DataIds to use to construct provenance. 

429 outputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

430 DataIds to use to populate the output calibration. 

431 

432 Returns 

433 ------- 

434 results : `lsst.pipe.base.Struct` 

435 The results struct containing: 

436 

437 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib` 

438 Final crosstalk calibration. 

439 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

440 Provenance data for the new calibration. 

441 

442 Raises 

443 ------ 

444 RuntimeError 

445 Raised if the input data contains multiple target detectors. 

446 

447 Notes 

448 ----- 

449 The lsstDebug.Info() method can be rewritten for __name__ = 

450 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

451 

452 debug.display['reduce'] : `bool` 

453 Display a histogram of the combined ratio measurements for 

454 a pair of source/target amplifiers from all input 

455 exposures/detectors. 

456 

457 """ 

458 if outputDims: 

459 calibChip = outputDims['detector'] 

460 instrument = outputDims['instrument'] 

461 else: 

462 # calibChip needs to be set manually in Gen2. 

463 calibChip = None 

464 instrument = None 

465 

466 self.log.info("Combining measurements from %d ratios and %d fluxes", 

467 len(inputRatios), len(inputFluxes) if inputFluxes else 0) 

468 

469 if inputFluxes is None: 

470 inputFluxes = [None for exp in inputRatios] 

471 

472 combinedRatios = defaultdict(lambda: defaultdict(list)) 

473 combinedFluxes = defaultdict(lambda: defaultdict(list)) 

474 for ratioDict, fluxDict in zip(inputRatios, inputFluxes): 

475 for targetChip in ratioDict: 

476 if calibChip and targetChip != calibChip: 

477 raise RuntimeError("Received multiple target chips!") 

478 

479 sourceChip = targetChip 

480 if sourceChip in ratioDict[targetChip]: 

481 ratios = ratioDict[targetChip][sourceChip] 

482 

483 for targetAmp in ratios: 

484 for sourceAmp in ratios[targetAmp]: 

485 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp]) 

486 if fluxDict: 

487 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp]) 

488 # TODO: DM-21904 

489 # Iterating over all other entries in ratioDict[targetChip] will yield 

490 # inter-chip terms. 

491 

492 for targetAmp in combinedRatios: 

493 for sourceAmp in combinedRatios[targetAmp]: 

494 self.log.info("Read %d pixels for %s -> %s", 

495 len(combinedRatios[targetAmp][sourceAmp]), 

496 targetAmp, sourceAmp) 

497 if len(combinedRatios[targetAmp][sourceAmp]) > 1: 

498 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp) 

499 

500 if self.config.fluxOrder == 0: 

501 self.log.info("Fitting crosstalk coefficients.") 

502 calib = self.measureCrosstalkCoefficients(combinedRatios, 

503 self.config.rejIter, self.config.rejSigma) 

504 else: 

505 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.") 

506 

507 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid)) 

508 

509 if self.config.doFiltering: 

510 # This step will apply the calculated validity values to 

511 # censor poorly measured coefficients. 

512 self.log.info("Filtering measured crosstalk to remove invalid solutions.") 

513 calib = self.filterCrosstalkCalib(calib) 

514 

515 # Populate the remainder of the calibration information. 

516 calib.hasCrosstalk = True 

517 calib.interChip = {} 

518 

519 # calibChip is the detector dimension, which is the detector Id 

520 calib._detectorId = calibChip 

521 if camera: 

522 calib._detectorName = camera[calibChip].getName() 

523 calib._detectorSerial = camera[calibChip].getSerial() 

524 

525 calib._instrument = instrument 

526 calib.updateMetadata(setCalibId=True, setDate=True) 

527 

528 # Make an IsrProvenance(). 

529 provenance = IsrProvenance(calibType="CROSSTALK") 

530 provenance._detectorName = calibChip 

531 if inputDims: 

532 provenance.fromDataIds(inputDims) 

533 provenance._instrument = instrument 

534 provenance.updateMetadata() 

535 

536 return pipeBase.Struct( 

537 outputCrosstalk=calib, 

538 outputProvenance=provenance, 

539 ) 

540 

541 def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma): 

542 """Measure crosstalk coefficients from the ratios. 

543 

544 Given a list of ratios for each target/source amp combination, 

545 we measure a sigma clipped mean and error. 

546 

547 The coefficient errors returned are the standard deviation of 

548 the final set of clipped input ratios. 

549 

550 Parameters 

551 ---------- 

552 ratios : `dict` of `dict` of `numpy.ndarray` 

553 Catalog of arrays of ratios. 

554 rejIter : `int` 

555 Number of rejection iterations. 

556 rejSigma : `float` 

557 Rejection threshold (sigma). 

558 

559 Returns 

560 ------- 

561 calib : `lsst.ip.isr.CrosstalkCalib` 

562 The output crosstalk calibration. 

563 

564 Notes 

565 ----- 

566 The lsstDebug.Info() method can be rewritten for __name__ = 

567 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

568 

569 debug.display['measure'] : `bool` 

570 Display the CDF of the combined ratio measurements for 

571 a pair of source/target amplifiers from the final set of 

572 clipped input ratios. 

573 """ 

574 calib = CrosstalkCalib(nAmp=len(ratios)) 

575 

576 # Calibration stores coefficients as a numpy ndarray. 

577 ordering = list(ratios.keys()) 

578 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)): 

579 if ii == jj: 

580 values = [0.0] 

581 else: 

582 values = np.array(ratios[ordering[ii]][ordering[jj]]) 

583 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

584 

585 calib.coeffNum[ii][jj] = len(values) 

586 

587 if len(values) == 0: 

588 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

589 calib.coeffs[ii][jj] = np.nan 

590 calib.coeffErr[ii][jj] = np.nan 

591 calib.coeffValid[ii][jj] = False 

592 else: 

593 if ii != jj: 

594 for rej in range(rejIter): 

595 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

596 sigma = 0.741*(hi - lo) 

597 good = np.abs(values - med) < rejSigma*sigma 

598 if good.sum() == len(good): 

599 break 

600 values = values[good] 

601 

602 calib.coeffs[ii][jj] = np.mean(values) 

603 if calib.coeffNum[ii][jj] == 1: 

604 calib.coeffErr[ii][jj] = np.nan 

605 else: 

606 correctionFactor = sigmaClipCorrection(rejSigma) 

607 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor 

608 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) 

609 > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj])) 

610 

611 if calib.coeffNum[ii][jj] > 1: 

612 self.debugRatios('measure', ratios, ordering[ii], ordering[jj], 

613 calib.coeffs[ii][jj], calib.coeffValid[ii][jj]) 

614 

615 return calib 

616 

617 @staticmethod 

618 def filterCrosstalkCalib(inCalib): 

619 """Apply valid constraints to the measured values. 

620 

621 Any measured coefficient that is determined to be invalid is 

622 set to zero, and has the error set to nan. The validation is 

623 determined by checking that the measured coefficient is larger 

624 than the calculated standard error of the mean. 

625 

626 Parameters 

627 ---------- 

628 inCalib : `lsst.ip.isr.CrosstalkCalib` 

629 Input calibration to filter. 

630 

631 Returns 

632 ------- 

633 outCalib : `lsst.ip.isr.CrosstalkCalib` 

634 Filtered calibration. 

635 """ 

636 outCalib = CrosstalkCalib() 

637 outCalib.numAmps = inCalib.numAmps 

638 

639 outCalib.coeffs = inCalib.coeffs 

640 outCalib.coeffs[~inCalib.coeffValid] = 0.0 

641 

642 outCalib.coeffErr = inCalib.coeffErr 

643 outCalib.coeffErr[~inCalib.coeffValid] = np.nan 

644 

645 outCalib.coeffNum = inCalib.coeffNum 

646 outCalib.coeffValid = inCalib.coeffValid 

647 

648 return outCalib 

649 

650 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False): 

651 """Utility function to examine the final CT ratio set. 

652 

653 Parameters 

654 ---------- 

655 stepname : `str` 

656 State of processing to view. 

657 ratios : `dict` of `dict` of `np.ndarray` 

658 Array of measured CT ratios, indexed by source/victim 

659 amplifier. 

660 i : `str` 

661 Index of the source amplifier. 

662 j : `str` 

663 Index of the target amplifier. 

664 coeff : `float`, optional 

665 Coefficient calculated to plot along with the simple mean. 

666 valid : `bool`, optional 

667 Validity to be added to the plot title. 

668 """ 

669 frame = getDebugFrame(self._display, stepname) 

670 if frame: 

671 if i == j or ratios is None or len(ratios) < 1: 

672 pass 

673 

674 ratioList = ratios[i][j] 

675 if ratioList is None or len(ratioList) < 1: 

676 pass 

677 

678 mean = np.mean(ratioList) 

679 std = np.std(ratioList) 

680 import matplotlib.pyplot as plt 

681 figure = plt.figure(1) 

682 figure.clear() 

683 plt.hist(x=ratioList, bins=len(ratioList), 

684 cumulative=True, color='b', density=True, histtype='step') 

685 plt.xlabel("Measured pixel ratio") 

686 plt.ylabel(f"CDF: n={len(ratioList)}") 

687 plt.xlim(np.percentile(ratioList, [1.0, 99])) 

688 plt.axvline(x=mean, color="k") 

689 plt.axvline(x=coeff, color='g') 

690 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r') 

691 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r') 

692 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}") 

693 figure.show() 

694 

695 prompt = "Press Enter to continue: " 

696 while True: 

697 ans = input(prompt).lower() 

698 if ans in ("", "c",): 

699 break 

700 elif ans in ("pdb", "p",): 

701 import pdb 

702 pdb.set_trace() 

703 plt.close() 

704 

705 

706class MeasureCrosstalkConfig(Config): 

707 extract = ConfigurableField( 

708 target=CrosstalkExtractTask, 

709 doc="Task to measure pixel ratios.", 

710 ) 

711 solver = ConfigurableField( 

712 target=CrosstalkSolveTask, 

713 doc="Task to convert ratio lists to crosstalk coefficients.", 

714 ) 

715 

716 

717class MeasureCrosstalkTask(pipeBase.CmdLineTask): 

718 """Measure intra-detector crosstalk. 

719 

720 See also 

721 -------- 

722 lsst.ip.isr.crosstalk.CrosstalkCalib 

723 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask 

724 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask 

725 

726 Notes 

727 ----- 

728 The crosstalk this method measures assumes that when a bright 

729 pixel is found in one detector amplifier, all other detector 

730 amplifiers may see a signal change in the same pixel location 

731 (relative to the readout amplifier) as these other pixels are read 

732 out at the same time. 

733 

734 After processing each input exposure through a limited set of ISR 

735 stages, bright unmasked pixels above the threshold are identified. 

736 The potential CT signal is found by taking the ratio of the 

737 appropriate background-subtracted pixel value on the other 

738 amplifiers to the input value on the source amplifier. If the 

739 source amplifier has a large number of bright pixels as well, the 

740 background level may be elevated, leading to poor ratio 

741 measurements. 

742 

743 The set of ratios found between each pair of amplifiers across all 

744 input exposures is then gathered to produce the final CT 

745 coefficients. The sigma-clipped mean and sigma are returned from 

746 these sets of ratios, with the coefficient to supply to the ISR 

747 CrosstalkTask() being the multiplicative inverse of these values. 

748 

749 This Task simply calls the pipetask versions of the measure 

750 crosstalk code. 

751 """ 

752 ConfigClass = MeasureCrosstalkConfig 

753 _DefaultName = "measureCrosstalk" 

754 

755 # Let's use this instead of messing with parseAndRun. 

756 RunnerClass = DataRefListRunner 

757 

758 def __init__(self, **kwargs): 

759 super().__init__(**kwargs) 

760 self.makeSubtask("extract") 

761 self.makeSubtask("solver") 

762 

763 def runDataRef(self, dataRefList): 

764 """Run extract task on each of inputs in the dataRef list, then pass 

765 that to the solver task. 

766 

767 Parameters 

768 ---------- 

769 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

770 Data references for exposures for detectors to process. 

771 

772 Returns 

773 ------- 

774 results : `lsst.pipe.base.Struct` 

775 The results struct containing: 

776 

777 ``outputCrosstalk`` : `lsst.ip.isr.CrosstalkCalib` 

778 Final crosstalk calibration. 

779 ``outputProvenance`` : `lsst.ip.isr.IsrProvenance` 

780 Provenance data for the new calibration. 

781 

782 Raises 

783 ------ 

784 RuntimeError 

785 Raised if multiple target detectors are supplied. 

786 """ 

787 dataRef = dataRefList[0] 

788 camera = dataRef.get("camera") 

789 

790 ratios = [] 

791 activeChip = None 

792 for dataRef in dataRefList: 

793 exposure = dataRef.get("postISRCCD") 

794 if activeChip: 

795 if exposure.getDetector().getName() != activeChip: 

796 raise RuntimeError("Too many input detectors supplied!") 

797 else: 

798 activeChip = exposure.getDetector().getName() 

799 

800 self.extract.debugView("extract", exposure) 

801 result = self.extract.run(exposure) 

802 ratios.append(result.outputRatios) 

803 

804 for detIter, detector in enumerate(camera): 

805 if detector.getName() == activeChip: 

806 detectorId = detIter 

807 outputDims = {'instrument': camera.getName(), 

808 'detector': detectorId, 

809 } 

810 

811 finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims) 

812 dataRef.put(finalResults.outputCrosstalk, "crosstalk") 

813 

814 return finalResults