Coverage for python/lsst/cp/pipe/measureCrosstalk.py: 15%

294 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:59 -0800

1# This file is part of cp_pipe 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import itertools 

22import numpy as np 

23 

24from collections import defaultdict 

25 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28 

29from lsstDebug import getDebugFrame 

30from lsst.afw.detection import FootprintSet, Threshold 

31from lsst.afw.display import getDisplay 

32from lsst.pex.config import Config, Field, ListField, ConfigurableField 

33from lsst.ip.isr import CrosstalkCalib, IsrProvenance 

34from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

35from lsst.cp.pipe.utils import (ddict2dict, sigmaClipCorrection) 

36 

37from ._lookupStaticCalibration import lookupStaticCalibration 

38 

39__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask", 

40 "CrosstalkSolveTask", "CrosstalkSolveConfig", 

41 "MeasureCrosstalkConfig", "MeasureCrosstalkTask"] 

42 

43 

44class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections, 

45 dimensions=("instrument", "exposure", "detector")): 

46 inputExp = cT.Input( 

47 name="crosstalkInputs", 

48 doc="Input post-ISR processed exposure to measure crosstalk from.", 

49 storageClass="Exposure", 

50 dimensions=("instrument", "exposure", "detector"), 

51 multiple=False, 

52 ) 

53 # TODO: Depends on DM-21904. 

54 sourceExp = cT.Input( 

55 name="crosstalkSource", 

56 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.", 

57 storageClass="Exposure", 

58 dimensions=("instrument", "exposure", "detector"), 

59 multiple=True, 

60 deferLoad=True, 

61 # lookupFunction=None, 

62 ) 

63 

64 outputRatios = cT.Output( 

65 name="crosstalkRatios", 

66 doc="Extracted crosstalk pixel ratios.", 

67 storageClass="StructuredDataDict", 

68 dimensions=("instrument", "exposure", "detector"), 

69 ) 

70 outputFluxes = cT.Output( 

71 name="crosstalkFluxes", 

72 doc="Source pixel fluxes used in ratios.", 

73 storageClass="StructuredDataDict", 

74 dimensions=("instrument", "exposure", "detector"), 

75 ) 

76 

77 def __init__(self, *, config=None): 

78 super().__init__(config=config) 

79 # Discard sourceExp until DM-21904 allows full interchip 

80 # measurements. 

81 self.inputs.discard("sourceExp") 

82 

83 

84class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=CrosstalkExtractConnections): 

86 """Configuration for the measurement of pixel ratios. 

87 """ 

88 

89 doMeasureInterchip = Field( 

90 dtype=bool, 

91 default=False, 

92 doc="Measure inter-chip crosstalk as well?", 

93 ) 

94 threshold = Field( 

95 dtype=float, 

96 default=30000, 

97 doc="Minimum level of source pixels for which to measure crosstalk." 

98 ) 

99 ignoreSaturatedPixels = Field( 

100 dtype=bool, 

101 default=True, 

102 doc="Should saturated pixels be ignored?" 

103 ) 

104 badMask = ListField( 

105 dtype=str, 

106 default=["BAD", "INTRP"], 

107 doc="Mask planes to ignore when identifying source pixels." 

108 ) 

109 isTrimmed = Field( 

110 dtype=bool, 

111 default=True, 

112 doc="Is the input exposure trimmed?" 

113 ) 

114 

115 def validate(self): 

116 super().validate() 

117 

118 # Ensure the handling of the SAT mask plane is consistent 

119 # with the ignoreSaturatedPixels value. 

120 if self.ignoreSaturatedPixels: 

121 if 'SAT' not in self.badMask: 

122 self.badMask.append('SAT') 

123 else: 

124 if 'SAT' in self.badMask: 

125 self.badMask = [mask for mask in self.badMask if mask != 'SAT'] 

126 

127 

128class CrosstalkExtractTask(pipeBase.PipelineTask, 

129 pipeBase.CmdLineTask): 

130 """Task to measure pixel ratios to find crosstalk. 

131 """ 

132 

133 ConfigClass = CrosstalkExtractConfig 

134 _DefaultName = 'cpCrosstalkExtract' 

135 

136 def run(self, inputExp, sourceExps=[]): 

137 """Measure pixel ratios between amplifiers in inputExp. 

138 

139 Extract crosstalk ratios between different amplifiers. 

140 

141 For pixels above ``config.threshold``, we calculate the ratio 

142 between each background-subtracted target amp and the source 

143 amp. We return a list of ratios for each pixel for each 

144 target/source combination, as nested dictionary containing the 

145 ratio. 

146 

147 Parameters 

148 ---------- 

149 inputExp : `lsst.afw.image.Exposure` 

150 Input exposure to measure pixel ratios on. 

151 sourceExp : `list` [`lsst.afw.image.Exposure`], optional 

152 List of chips to use as sources to measure inter-chip 

153 crosstalk. 

154 

155 Returns 

156 ------- 

157 results : `lsst.pipe.base.Struct` 

158 The results struct containing: 

159 

160 ``outputRatios`` 

161 A catalog of ratio lists. The dictionaries are 

162 indexed such that: 

163 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp] 

164 contains the ratio list for that combination (`dict` 

165 [`dict` [`dict` [`dict` [`list`]]]]). 

166 ``outputFluxes`` 

167 A catalog of flux lists. The dictionaries are 

168 indexed such that: 

169 outputFluxes[sourceChip][sourceAmp] contains the flux 

170 list used in the outputRatios (`dict` [`dict` 

171 [`list`]]). 

172 """ 

173 outputRatios = defaultdict(lambda: defaultdict(dict)) 

174 outputFluxes = defaultdict(lambda: defaultdict(dict)) 

175 

176 threshold = self.config.threshold 

177 badPixels = list(self.config.badMask) 

178 

179 targetDetector = inputExp.getDetector() 

180 targetChip = targetDetector.getName() 

181 

182 # Always look at the target chip first, then go to any other 

183 # supplied exposures. 

184 sourceExtractExps = [inputExp] 

185 sourceExtractExps.extend(sourceExps) 

186 

187 self.log.info("Measuring full detector background for target: %s", targetChip) 

188 targetIm = inputExp.getMaskedImage() 

189 FootprintSet(targetIm, Threshold(threshold), "DETECTED") 

190 detected = targetIm.getMask().getPlaneBitMask("DETECTED") 

191 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) 

192 

193 self.debugView('extract', inputExp) 

194 

195 for sourceExp in sourceExtractExps: 

196 sourceDetector = sourceExp.getDetector() 

197 sourceChip = sourceDetector.getName() 

198 sourceIm = sourceExp.getMaskedImage() 

199 bad = sourceIm.getMask().getPlaneBitMask(badPixels) 

200 self.log.info("Measuring crosstalk from source: %s", sourceChip) 

201 

202 if sourceExp != inputExp: 

203 FootprintSet(sourceIm, Threshold(threshold), "DETECTED") 

204 detected = sourceIm.getMask().getPlaneBitMask("DETECTED") 

205 

206 # The dictionary of amp-to-amp ratios for this pair of 

207 # source->target detectors. 

208 ratioDict = defaultdict(lambda: defaultdict(list)) 

209 extractedCount = 0 

210 

211 for sourceAmp in sourceDetector: 

212 sourceAmpName = sourceAmp.getName() 

213 sourceAmpBBox = sourceAmp.getBBox() if self.config.isTrimmed else sourceAmp.getRawDataBBox() 

214 sourceAmpImage = sourceIm[sourceAmpBBox] 

215 sourceMask = sourceAmpImage.mask.array 

216 select = ((sourceMask & detected > 0) 

217 & (sourceMask & bad == 0) 

218 & np.isfinite(sourceAmpImage.image.array)) 

219 count = np.sum(select) 

220 self.log.debug(" Source amplifier: %s", sourceAmpName) 

221 

222 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist() 

223 

224 for targetAmp in targetDetector: 

225 # iterate over targetExposure 

226 targetAmpName = targetAmp.getName() 

227 if sourceAmpName == targetAmpName and sourceChip == targetChip: 

228 ratioDict[sourceAmpName][targetAmpName] = [] 

229 continue 

230 self.log.debug(" Target amplifier: %s", targetAmpName) 

231 

232 targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image, 

233 targetAmp, sourceAmp, 

234 isTrimmed=self.config.isTrimmed) 

235 ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select] 

236 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist() 

237 extractedCount += count 

238 

239 self.debugPixels('pixels', 

240 sourceAmpImage.image.array[select], 

241 targetAmpImage.array[select] - bg, 

242 sourceAmpName, targetAmpName) 

243 

244 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)", 

245 extractedCount, sourceChip, targetChip, bg) 

246 outputRatios[targetChip][sourceChip] = ratioDict 

247 

248 return pipeBase.Struct( 

249 outputRatios=ddict2dict(outputRatios), 

250 outputFluxes=ddict2dict(outputFluxes) 

251 ) 

252 

253 def debugView(self, stepname, exposure): 

254 """Utility function to examine the image being processed. 

255 

256 Parameters 

257 ---------- 

258 stepname : `str` 

259 State of processing to view. 

260 exposure : `lsst.afw.image.Exposure` 

261 Exposure to view. 

262 """ 

263 frame = getDebugFrame(self._display, stepname) 

264 if frame: 

265 display = getDisplay(frame) 

266 display.scale('asinh', 'zscale') 

267 display.mtv(exposure) 

268 

269 prompt = "Press Enter to continue: " 

270 while True: 

271 ans = input(prompt).lower() 

272 if ans in ("", "c",): 

273 break 

274 

275 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName): 

276 """Utility function to examine the CT ratio pixel values. 

277 

278 Parameters 

279 ---------- 

280 stepname : `str` 

281 State of processing to view. 

282 pixelsIn : `np.ndarray`, (N,) 

283 Pixel values from the potential crosstalk source. 

284 pixelsOut : `np.ndarray`, (N,) 

285 Pixel values from the potential crosstalk target. 

286 sourceName : `str` 

287 Source amplifier name 

288 targetName : `str` 

289 Target amplifier name 

290 """ 

291 frame = getDebugFrame(self._display, stepname) 

292 if frame: 

293 import matplotlib.pyplot as plt 

294 figure = plt.figure(1) 

295 figure.clear() 

296 

297 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

298 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+') 

299 plt.xlabel("Source amplifier pixel value") 

300 plt.ylabel("Measured pixel ratio") 

301 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: " 

302 f"{(np.median(pixelsOut / pixelsIn))}") 

303 figure.show() 

304 

305 prompt = "Press Enter to continue: " 

306 while True: 

307 ans = input(prompt).lower() 

308 if ans in ("", "c",): 

309 break 

310 plt.close() 

311 

312 

313class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections, 

314 dimensions=("instrument", "detector")): 

315 inputRatios = cT.Input( 

316 name="crosstalkRatios", 

317 doc="Ratios measured for an input exposure.", 

318 storageClass="StructuredDataDict", 

319 dimensions=("instrument", "exposure", "detector"), 

320 multiple=True, 

321 ) 

322 inputFluxes = cT.Input( 

323 name="crosstalkFluxes", 

324 doc="Fluxes of CT source pixels, for nonlinear fits.", 

325 storageClass="StructuredDataDict", 

326 dimensions=("instrument", "exposure", "detector"), 

327 multiple=True, 

328 ) 

329 camera = cT.PrerequisiteInput( 

330 name="camera", 

331 doc="Camera the input data comes from.", 

332 storageClass="Camera", 

333 dimensions=("instrument",), 

334 isCalibration=True, 

335 lookupFunction=lookupStaticCalibration, 

336 ) 

337 

338 outputCrosstalk = cT.Output( 

339 name="crosstalk", 

340 doc="Output proposed crosstalk calibration.", 

341 storageClass="CrosstalkCalib", 

342 dimensions=("instrument", "detector"), 

343 multiple=False, 

344 isCalibration=True, 

345 ) 

346 

347 def __init__(self, *, config=None): 

348 super().__init__(config=config) 

349 

350 if config.fluxOrder == 0: 

351 self.inputs.discard("inputFluxes") 

352 

353 

354class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig, 

355 pipelineConnections=CrosstalkSolveConnections): 

356 """Configuration for the solving of crosstalk from pixel ratios. 

357 """ 

358 

359 rejIter = Field( 

360 dtype=int, 

361 default=3, 

362 doc="Number of rejection iterations for final coefficient calculation.", 

363 ) 

364 rejSigma = Field( 

365 dtype=float, 

366 default=2.0, 

367 doc="Rejection threshold (sigma) for final coefficient calculation.", 

368 ) 

369 fluxOrder = Field( 

370 dtype=int, 

371 default=0, 

372 doc="Polynomial order in source flux to fit crosstalk.", 

373 ) 

374 doFiltering = Field( 

375 dtype=bool, 

376 default=False, 

377 doc="Filter generated crosstalk to remove marginal measurements.", 

378 ) 

379 

380 

381class CrosstalkSolveTask(pipeBase.PipelineTask, 

382 pipeBase.CmdLineTask): 

383 """Task to solve crosstalk from pixel ratios. 

384 """ 

385 

386 ConfigClass = CrosstalkSolveConfig 

387 _DefaultName = 'cpCrosstalkSolve' 

388 

389 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

390 """Ensure that the input and output dimensions are passed along. 

391 

392 Parameters 

393 ---------- 

394 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

395 Butler to operate on. 

396 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

397 Input data refs to load. 

398 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

399 Output data refs to persist. 

400 """ 

401 inputs = butlerQC.get(inputRefs) 

402 

403 # Use the dimensions to set calib/provenance information. 

404 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios] 

405 inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName() 

406 

407 outputs = self.run(**inputs) 

408 butlerQC.put(outputs, outputRefs) 

409 

410 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None): 

411 """Combine ratios to produce crosstalk coefficients. 

412 

413 Parameters 

414 ---------- 

415 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]] 

416 A list of nested dictionaries of ratios indexed by target 

417 and source chip, then by target and source amplifier. 

418 inputFluxes : `list` [`dict` [`dict` [`list`]]] 

419 A list of nested dictionaries of source pixel fluxes, indexed 

420 by source chip and amplifier. 

421 camera : `lsst.afw.cameraGeom.Camera` 

422 Input camera. 

423 inputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

424 DataIds to use to construct provenance. 

425 outputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

426 DataIds to use to populate the output calibration. 

427 

428 Returns 

429 ------- 

430 results : `lsst.pipe.base.Struct` 

431 The results struct containing: 

432 

433 ``outputCrosstalk`` 

434 Final crosstalk calibration 

435 (`lsst.ip.isr.CrosstalkCalib`). 

436 ``outputProvenance`` 

437 Provenance data for the new calibration 

438 (`lsst.ip.isr.IsrProvenance`). 

439 

440 Raises 

441 ------ 

442 RuntimeError 

443 Raised if the input data contains multiple target detectors. 

444 """ 

445 if outputDims: 

446 calibChip = outputDims['detector'] 

447 instrument = outputDims['instrument'] 

448 else: 

449 # calibChip needs to be set manually in Gen2. 

450 calibChip = None 

451 instrument = None 

452 

453 if camera and calibChip: 

454 calibDetector = camera[calibChip] 

455 else: 

456 calibDetector = None 

457 

458 self.log.info("Combining measurements from %d ratios and %d fluxes", 

459 len(inputRatios), len(inputFluxes) if inputFluxes else 0) 

460 

461 if inputFluxes is None: 

462 inputFluxes = [None for exp in inputRatios] 

463 

464 combinedRatios = defaultdict(lambda: defaultdict(list)) 

465 combinedFluxes = defaultdict(lambda: defaultdict(list)) 

466 for ratioDict, fluxDict in zip(inputRatios, inputFluxes): 

467 for targetChip in ratioDict: 

468 if calibChip and targetChip != calibChip and targetChip != calibDetector.getName(): 

469 raise RuntimeError(f"Target chip: {targetChip} does not match calibration dimension: " 

470 f"{calibChip}, {calibDetector.getName()}!") 

471 

472 sourceChip = targetChip 

473 if sourceChip in ratioDict[targetChip]: 

474 ratios = ratioDict[targetChip][sourceChip] 

475 

476 for targetAmp in ratios: 

477 for sourceAmp in ratios[targetAmp]: 

478 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp]) 

479 if fluxDict: 

480 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp]) 

481 # TODO: DM-21904 

482 # Iterating over all other entries in 

483 # ratioDict[targetChip] will yield inter-chip terms. 

484 

485 for targetAmp in combinedRatios: 

486 for sourceAmp in combinedRatios[targetAmp]: 

487 self.log.info("Read %d pixels for %s -> %s", 

488 len(combinedRatios[targetAmp][sourceAmp]), 

489 targetAmp, sourceAmp) 

490 if len(combinedRatios[targetAmp][sourceAmp]) > 1: 

491 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp) 

492 

493 if self.config.fluxOrder == 0: 

494 self.log.info("Fitting crosstalk coefficients.") 

495 calib = self.measureCrosstalkCoefficients(combinedRatios, 

496 self.config.rejIter, self.config.rejSigma) 

497 else: 

498 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.") 

499 

500 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid)) 

501 

502 if self.config.doFiltering: 

503 # This step will apply the calculated validity values to 

504 # censor poorly measured coefficients. 

505 self.log.info("Filtering measured crosstalk to remove invalid solutions.") 

506 calib = self.filterCrosstalkCalib(calib) 

507 

508 # Populate the remainder of the calibration information. 

509 calib.hasCrosstalk = True 

510 calib.interChip = {} 

511 

512 # calibChip is the detector dimension, which is the detector Id 

513 calib._detectorId = calibChip 

514 if calibDetector: 

515 calib._detectorName = calibDetector.getName() 

516 calib._detectorSerial = calibDetector.getSerial() 

517 

518 calib._instrument = instrument 

519 calib.updateMetadata(setCalibId=True, setDate=True) 

520 

521 # Make an IsrProvenance(). 

522 provenance = IsrProvenance(calibType="CROSSTALK") 

523 provenance._detectorName = calibChip 

524 if inputDims: 

525 provenance.fromDataIds(inputDims) 

526 provenance._instrument = instrument 

527 provenance.updateMetadata() 

528 

529 return pipeBase.Struct( 

530 outputCrosstalk=calib, 

531 outputProvenance=provenance, 

532 ) 

533 

534 def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma): 

535 """Measure crosstalk coefficients from the ratios. 

536 

537 Given a list of ratios for each target/source amp combination, 

538 we measure a sigma clipped mean and error. 

539 

540 The coefficient errors returned are the standard deviation of 

541 the final set of clipped input ratios. 

542 

543 Parameters 

544 ---------- 

545 ratios : `dict` [`dict` [`numpy.ndarray`]] 

546 Catalog of arrays of ratios. The ratio arrays are one-dimensional 

547 rejIter : `int` 

548 Number of rejection iterations. 

549 rejSigma : `float` 

550 Rejection threshold (sigma). 

551 

552 Returns 

553 ------- 

554 calib : `lsst.ip.isr.CrosstalkCalib` 

555 The output crosstalk calibration. 

556 """ 

557 calib = CrosstalkCalib(nAmp=len(ratios)) 

558 

559 # Calibration stores coefficients as a numpy ndarray. 

560 ordering = list(ratios.keys()) 

561 for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)): 

562 if ii == jj: 

563 values = [0.0] 

564 else: 

565 values = np.array(ratios[ordering[ii]][ordering[jj]]) 

566 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

567 

568 calib.coeffNum[ii][jj] = len(values) 

569 

570 if len(values) == 0: 

571 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

572 calib.coeffs[ii][jj] = np.nan 

573 calib.coeffErr[ii][jj] = np.nan 

574 calib.coeffValid[ii][jj] = False 

575 else: 

576 if ii != jj: 

577 for rej in range(rejIter): 

578 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

579 sigma = 0.741*(hi - lo) 

580 good = np.abs(values - med) < rejSigma*sigma 

581 if good.sum() == len(good): 

582 break 

583 values = values[good] 

584 

585 calib.coeffs[ii][jj] = np.mean(values) 

586 if calib.coeffNum[ii][jj] == 1: 

587 calib.coeffErr[ii][jj] = np.nan 

588 else: 

589 correctionFactor = sigmaClipCorrection(rejSigma) 

590 calib.coeffErr[ii][jj] = np.std(values) * correctionFactor 

591 calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) 

592 > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj])) 

593 

594 if calib.coeffNum[ii][jj] > 1: 

595 self.debugRatios('measure', ratios, ordering[ii], ordering[jj], 

596 calib.coeffs[ii][jj], calib.coeffValid[ii][jj]) 

597 

598 return calib 

599 

600 @staticmethod 

601 def filterCrosstalkCalib(inCalib): 

602 """Apply valid constraints to the measured values. 

603 

604 Any measured coefficient that is determined to be invalid is 

605 set to zero, and has the error set to nan. The validation is 

606 determined by checking that the measured coefficient is larger 

607 than the calculated standard error of the mean. 

608 

609 Parameters 

610 ---------- 

611 inCalib : `lsst.ip.isr.CrosstalkCalib` 

612 Input calibration to filter. 

613 

614 Returns 

615 ------- 

616 outCalib : `lsst.ip.isr.CrosstalkCalib` 

617 Filtered calibration. 

618 """ 

619 outCalib = CrosstalkCalib() 

620 outCalib.numAmps = inCalib.numAmps 

621 

622 outCalib.coeffs = inCalib.coeffs 

623 outCalib.coeffs[~inCalib.coeffValid] = 0.0 

624 

625 outCalib.coeffErr = inCalib.coeffErr 

626 outCalib.coeffErr[~inCalib.coeffValid] = np.nan 

627 

628 outCalib.coeffNum = inCalib.coeffNum 

629 outCalib.coeffValid = inCalib.coeffValid 

630 

631 return outCalib 

632 

633 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False): 

634 """Utility function to examine the final CT ratio set. 

635 

636 Parameters 

637 ---------- 

638 stepname : `str` 

639 State of processing to view. 

640 ratios : `dict` [`dict` [`numpy.ndarray`]] 

641 Array of measured CT ratios, indexed by source/victim 

642 amplifier. These arrays are one-dimensional. 

643 i : `str` 

644 Index of the source amplifier. 

645 j : `str` 

646 Index of the target amplifier. 

647 coeff : `float`, optional 

648 Coefficient calculated to plot along with the simple mean. 

649 valid : `bool`, optional 

650 Validity to be added to the plot title. 

651 """ 

652 frame = getDebugFrame(self._display, stepname) 

653 if frame: 

654 if i == j or ratios is None or len(ratios) < 1: 

655 pass 

656 

657 ratioList = ratios[i][j] 

658 if ratioList is None or len(ratioList) < 1: 

659 pass 

660 

661 mean = np.mean(ratioList) 

662 std = np.std(ratioList) 

663 import matplotlib.pyplot as plt 

664 figure = plt.figure(1) 

665 figure.clear() 

666 plt.hist(x=ratioList, bins=len(ratioList), 

667 cumulative=True, color='b', density=True, histtype='step') 

668 plt.xlabel("Measured pixel ratio") 

669 plt.ylabel(f"CDF: n={len(ratioList)}") 

670 plt.xlim(np.percentile(ratioList, [1.0, 99])) 

671 plt.axvline(x=mean, color="k") 

672 plt.axvline(x=coeff, color='g') 

673 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r') 

674 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r') 

675 plt.title(f"(Source {i} -> Target {j}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}") 

676 figure.show() 

677 

678 prompt = "Press Enter to continue: " 

679 while True: 

680 ans = input(prompt).lower() 

681 if ans in ("", "c",): 

682 break 

683 elif ans in ("pdb", "p",): 

684 import pdb 

685 pdb.set_trace() 

686 plt.close() 

687 

688 

689class MeasureCrosstalkConfig(Config): 

690 extract = ConfigurableField( 

691 target=CrosstalkExtractTask, 

692 doc="Task to measure pixel ratios.", 

693 ) 

694 solver = ConfigurableField( 

695 target=CrosstalkSolveTask, 

696 doc="Task to convert ratio lists to crosstalk coefficients.", 

697 ) 

698 

699 

700class MeasureCrosstalkTask(pipeBase.CmdLineTask): 

701 """Measure intra-detector crosstalk. 

702 

703 See also 

704 -------- 

705 lsst.ip.isr.crosstalk.CrosstalkCalib 

706 lsst.cp.pipe.measureCrosstalk.CrosstalkExtractTask 

707 lsst.cp.pipe.measureCrosstalk.CrosstalkSolveTask 

708 

709 Notes 

710 ----- 

711 The crosstalk this method measures assumes that when a bright 

712 pixel is found in one detector amplifier, all other detector 

713 amplifiers may see a signal change in the same pixel location 

714 (relative to the readout amplifier) as these other pixels are read 

715 out at the same time. 

716 

717 After processing each input exposure through a limited set of ISR 

718 stages, bright unmasked pixels above the threshold are identified. 

719 The potential CT signal is found by taking the ratio of the 

720 appropriate background-subtracted pixel value on the other 

721 amplifiers to the input value on the source amplifier. If the 

722 source amplifier has a large number of bright pixels as well, the 

723 background level may be elevated, leading to poor ratio 

724 measurements. 

725 

726 The set of ratios found between each pair of amplifiers across all 

727 input exposures is then gathered to produce the final CT 

728 coefficients. The sigma-clipped mean and sigma are returned from 

729 these sets of ratios, with the coefficient to supply to the ISR 

730 CrosstalkTask() being the multiplicative inverse of these values. 

731 

732 This Task simply calls the pipetask versions of the measure 

733 crosstalk code. 

734 """ 

735 

736 ConfigClass = MeasureCrosstalkConfig 

737 _DefaultName = "measureCrosstalk" 

738 

739 # Let's use this instead of messing with parseAndRun. 

740 RunnerClass = DataRefListRunner 

741 

742 def __init__(self, **kwargs): 

743 super().__init__(**kwargs) 

744 self.makeSubtask("extract") 

745 self.makeSubtask("solver") 

746 

747 def runDataRef(self, dataRefList): 

748 """Run extract task on each of inputs in the dataRef list, then pass 

749 that to the solver task. 

750 

751 Parameters 

752 ---------- 

753 dataRefList : `list` [`lsst.daf.peristence.ButlerDataRef`] 

754 Data references for exposures for detectors to process. 

755 

756 Returns 

757 ------- 

758 results : `lsst.pipe.base.Struct` 

759 The results struct containing: 

760 

761 ``outputCrosstalk`` 

762 Final crosstalk calibration 

763 (`lsst.ip.isr.CrosstalkCalib`). 

764 ``outputProvenance`` 

765 Provenance data for the new calibration 

766 (`lsst.ip.isr.IsrProvenance`). 

767 

768 Raises 

769 ------ 

770 RuntimeError 

771 Raised if multiple target detectors are supplied. 

772 """ 

773 dataRef = dataRefList[0] 

774 camera = dataRef.get("camera") 

775 

776 ratios = [] 

777 activeChip = None 

778 for dataRef in dataRefList: 

779 exposure = dataRef.get("postISRCCD") 

780 if activeChip: 

781 if exposure.getDetector().getName() != activeChip: 

782 raise RuntimeError("Too many input detectors supplied!") 

783 else: 

784 activeChip = exposure.getDetector().getName() 

785 

786 self.extract.debugView("extract", exposure) 

787 result = self.extract.run(exposure) 

788 ratios.append(result.outputRatios) 

789 

790 for detIter, detector in enumerate(camera): 

791 if detector.getName() == activeChip: 

792 detectorId = detIter 

793 outputDims = {'instrument': camera.getName(), 

794 'detector': detectorId, 

795 } 

796 

797 finalResults = self.solver.run(ratios, camera=camera, outputDims=outputDims) 

798 dataRef.put(finalResults.outputCrosstalk, "crosstalk") 

799 

800 return finalResults