Coverage for python/lsst/cp/pipe/measureCrosstalk.py: 14%

291 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-26 01:39 -0700

1# This file is part of cp_pipe 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import itertools 

22import numpy as np 

23 

24from collections import defaultdict 

25 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28 

29from lsstDebug import getDebugFrame 

30from lsst.afw.detection import FootprintSet, Threshold 

31from lsst.afw.display import getDisplay 

32from lsst.pex.config import ConfigurableField, Field, ListField 

33from lsst.ip.isr import CrosstalkCalib, IsrProvenance 

34from lsst.cp.pipe.utils import (ddict2dict, sigmaClipCorrection) 

35from lsst.meas.algorithms import SubtractBackgroundTask 

36 

37from ._lookupStaticCalibration import lookupStaticCalibration 

38 

39__all__ = ["CrosstalkExtractConfig", "CrosstalkExtractTask", 

40 "CrosstalkSolveTask", "CrosstalkSolveConfig"] 

41 

42 

43class CrosstalkExtractConnections(pipeBase.PipelineTaskConnections, 

44 dimensions=("instrument", "exposure", "detector")): 

45 inputExp = cT.Input( 

46 name="crosstalkInputs", 

47 doc="Input post-ISR processed exposure to measure crosstalk from.", 

48 storageClass="Exposure", 

49 dimensions=("instrument", "exposure", "detector"), 

50 multiple=False, 

51 ) 

52 # TODO: Depends on DM-21904. 

53 sourceExp = cT.Input( 

54 name="crosstalkSource", 

55 doc="Post-ISR exposure to measure for inter-chip crosstalk onto inputExp.", 

56 storageClass="Exposure", 

57 dimensions=("instrument", "exposure", "detector"), 

58 multiple=True, 

59 deferLoad=True, 

60 # lookupFunction=None, 

61 ) 

62 

63 outputRatios = cT.Output( 

64 name="crosstalkRatios", 

65 doc="Extracted crosstalk pixel ratios.", 

66 storageClass="StructuredDataDict", 

67 dimensions=("instrument", "exposure", "detector"), 

68 ) 

69 outputFluxes = cT.Output( 

70 name="crosstalkFluxes", 

71 doc="Source pixel fluxes used in ratios.", 

72 storageClass="StructuredDataDict", 

73 dimensions=("instrument", "exposure", "detector"), 

74 ) 

75 

76 def __init__(self, *, config=None): 

77 super().__init__(config=config) 

78 # Discard sourceExp until DM-21904 allows full interchip 

79 # measurements. 

80 self.inputs.discard("sourceExp") 

81 

82 

83class CrosstalkExtractConfig(pipeBase.PipelineTaskConfig, 

84 pipelineConnections=CrosstalkExtractConnections): 

85 """Configuration for the measurement of pixel ratios. 

86 """ 

87 

88 doMeasureInterchip = Field( 

89 dtype=bool, 

90 default=False, 

91 doc="Measure inter-chip crosstalk as well?", 

92 ) 

93 threshold = Field( 

94 dtype=float, 

95 default=30000, 

96 doc="Minimum level of source pixels for which to measure crosstalk." 

97 ) 

98 ignoreSaturatedPixels = Field( 

99 dtype=bool, 

100 default=False, 

101 doc="Should saturated pixels be ignored?" 

102 ) 

103 badMask = ListField( 

104 dtype=str, 

105 default=["BAD", "INTRP"], 

106 doc="Mask planes to ignore when identifying source pixels." 

107 ) 

108 isTrimmed = Field( 

109 dtype=bool, 

110 default=True, 

111 doc="Is the input exposure trimmed?" 

112 ) 

113 background = ConfigurableField( 

114 target=SubtractBackgroundTask, 

115 doc="Background estimation task.", 

116 ) 

117 

118 def validate(self): 

119 super().validate() 

120 

121 # Ensure the handling of the SAT mask plane is consistent 

122 # with the ignoreSaturatedPixels value. 

123 if self.ignoreSaturatedPixels: 

124 if 'SAT' not in self.badMask: 

125 self.badMask.append('SAT') 

126 else: 

127 if 'SAT' in self.badMask: 

128 self.badMask = [mask for mask in self.badMask if mask != 'SAT'] 

129 

130 

131class CrosstalkExtractTask(pipeBase.PipelineTask): 

132 """Task to measure pixel ratios to find crosstalk. 

133 """ 

134 

135 ConfigClass = CrosstalkExtractConfig 

136 _DefaultName = 'cpCrosstalkExtract' 

137 

138 def __init__(self, **kwargs): 

139 super().__init__(**kwargs) 

140 self.makeSubtask("background") 

141 

142 def run(self, inputExp, sourceExps=[]): 

143 """Measure pixel ratios between amplifiers in inputExp. 

144 

145 Extract crosstalk ratios between different amplifiers. 

146 

147 For pixels above ``config.threshold``, we calculate the ratio 

148 between each background-subtracted target amp and the source 

149 amp. We return a list of ratios for each pixel for each 

150 target/source combination, as nested dictionary containing the 

151 ratio. 

152 

153 Parameters 

154 ---------- 

155 inputExp : `lsst.afw.image.Exposure` 

156 Input exposure to measure pixel ratios on. 

157 sourceExp : `list` [`lsst.afw.image.Exposure`], optional 

158 List of chips to use as sources to measure inter-chip 

159 crosstalk. 

160 

161 Returns 

162 ------- 

163 results : `lsst.pipe.base.Struct` 

164 The results struct containing: 

165 

166 ``outputRatios`` 

167 A catalog of ratio lists. The dictionaries are 

168 indexed such that: 

169 outputRatios[targetChip][sourceChip][targetAmp][sourceAmp] 

170 contains the ratio list for that combination (`dict` 

171 [`dict` [`dict` [`dict` [`list`]]]]). 

172 ``outputFluxes`` 

173 A catalog of flux lists. The dictionaries are 

174 indexed such that: 

175 outputFluxes[sourceChip][sourceAmp] contains the flux 

176 list used in the outputRatios (`dict` [`dict` 

177 [`list`]]). 

178 """ 

179 outputRatios = defaultdict(lambda: defaultdict(dict)) 

180 outputFluxes = defaultdict(lambda: defaultdict(dict)) 

181 

182 threshold = self.config.threshold 

183 badPixels = list(self.config.badMask) 

184 

185 targetDetector = inputExp.getDetector() 

186 targetChip = targetDetector.getName() 

187 

188 # Always look at the target chip first, then go to any other 

189 # supplied exposures. 

190 sourceExtractExps = [inputExp] 

191 sourceExtractExps.extend(sourceExps) 

192 

193 self.log.info("Measuring full detector background for target: %s", targetChip) 

194 targetIm = inputExp.getMaskedImage() 

195 FootprintSet(targetIm, Threshold(threshold), "DETECTED") 

196 detected = targetIm.getMask().getPlaneBitMask("DETECTED") 

197 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) 

198 backgroundModel = self.background.fitBackground(inputExp.maskedImage) 

199 backgroundIm = backgroundModel.getImageF() 

200 

201 self.debugView('extract', inputExp) 

202 

203 for sourceExp in sourceExtractExps: 

204 sourceDetector = sourceExp.getDetector() 

205 sourceChip = sourceDetector.getName() 

206 sourceIm = sourceExp.getMaskedImage() 

207 bad = sourceIm.getMask().getPlaneBitMask(badPixels) 

208 self.log.info("Measuring crosstalk from source: %s", sourceChip) 

209 

210 if sourceExp != inputExp: 

211 FootprintSet(sourceIm, Threshold(threshold), "DETECTED") 

212 detected = sourceIm.getMask().getPlaneBitMask("DETECTED") 

213 

214 # The dictionary of amp-to-amp ratios for this pair of 

215 # source->target detectors. 

216 ratioDict = defaultdict(lambda: defaultdict(list)) 

217 extractedCount = 0 

218 

219 for sourceAmp in sourceDetector: 

220 sourceAmpName = sourceAmp.getName() 

221 sourceAmpBBox = sourceAmp.getBBox() if self.config.isTrimmed else sourceAmp.getRawDataBBox() 

222 sourceAmpImage = sourceIm[sourceAmpBBox] 

223 sourceMask = sourceAmpImage.mask.array 

224 select = ((sourceMask & detected > 0) 

225 & (sourceMask & bad == 0) 

226 & np.isfinite(sourceAmpImage.image.array)) 

227 count = np.sum(select) 

228 self.log.debug(" Source amplifier: %s", sourceAmpName) 

229 

230 outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist() 

231 

232 for targetAmp in targetDetector: 

233 # iterate over targetExposure 

234 targetAmpName = targetAmp.getName() 

235 if sourceAmpName == targetAmpName and sourceChip == targetChip: 

236 ratioDict[targetAmpName][sourceAmpName] = [] 

237 continue 

238 

239 self.log.debug(" Target amplifier: %s", targetAmpName) 

240 

241 targetAmpImage = CrosstalkCalib.extractAmp(targetIm, 

242 targetAmp, sourceAmp, 

243 isTrimmed=self.config.isTrimmed) 

244 targetBkgImage = CrosstalkCalib.extractAmp(backgroundIm, 

245 targetAmp, sourceAmp, 

246 isTrimmed=self.config.isTrimmed) 

247 

248 bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) 

249 

250 ratios = ((targetAmpImage.image.array[select] - targetBkgImage.array[select]) 

251 / sourceAmpImage.image.array[select]) 

252 

253 ratioDict[targetAmpName][sourceAmpName] = ratios.tolist() 

254 self.log.info("Amp extracted %d pixels from %s -> %s", 

255 count, sourceAmpName, targetAmpName) 

256 extractedCount += count 

257 

258 self.debugPixels('pixels', 

259 sourceAmpImage.image.array[select], 

260 targetAmpImage.image.array[select] - bg, 

261 sourceAmpName, targetAmpName) 

262 

263 self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)", 

264 extractedCount, sourceChip, targetChip, bg) 

265 outputRatios[targetChip][sourceChip] = ratioDict 

266 

267 return pipeBase.Struct( 

268 outputRatios=ddict2dict(outputRatios), 

269 outputFluxes=ddict2dict(outputFluxes) 

270 ) 

271 

272 def debugView(self, stepname, exposure): 

273 """Utility function to examine the image being processed. 

274 

275 Parameters 

276 ---------- 

277 stepname : `str` 

278 State of processing to view. 

279 exposure : `lsst.afw.image.Exposure` 

280 Exposure to view. 

281 """ 

282 frame = getDebugFrame(self._display, stepname) 

283 if frame: 

284 display = getDisplay(frame) 

285 display.scale('asinh', 'zscale') 

286 display.mtv(exposure) 

287 

288 prompt = "Press Enter to continue: " 

289 while True: 

290 ans = input(prompt).lower() 

291 if ans in ("", "c",): 

292 break 

293 

294 def debugPixels(self, stepname, pixelsIn, pixelsOut, sourceName, targetName): 

295 """Utility function to examine the CT ratio pixel values. 

296 

297 Parameters 

298 ---------- 

299 stepname : `str` 

300 State of processing to view. 

301 pixelsIn : `np.ndarray`, (N,) 

302 Pixel values from the potential crosstalk source. 

303 pixelsOut : `np.ndarray`, (N,) 

304 Pixel values from the potential crosstalk target. 

305 sourceName : `str` 

306 Source amplifier name 

307 targetName : `str` 

308 Target amplifier name 

309 """ 

310 frame = getDebugFrame(self._display, stepname) 

311 if frame: 

312 import matplotlib.pyplot as plt 

313 figure = plt.figure(1) 

314 figure.clear() 

315 

316 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

317 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+') 

318 plt.xlabel("Source amplifier pixel value") 

319 plt.ylabel("Measured pixel ratio") 

320 plt.title(f"(Source {sourceName} -> Target {targetName}) median ratio: " 

321 f"{(np.median(pixelsOut / pixelsIn))}") 

322 figure.show() 

323 

324 prompt = "Press Enter to continue: " 

325 while True: 

326 ans = input(prompt).lower() 

327 if ans in ("", "c",): 

328 break 

329 plt.close() 

330 

331 

332class CrosstalkSolveConnections(pipeBase.PipelineTaskConnections, 

333 dimensions=("instrument", "detector")): 

334 inputRatios = cT.Input( 

335 name="crosstalkRatios", 

336 doc="Ratios measured for an input exposure.", 

337 storageClass="StructuredDataDict", 

338 dimensions=("instrument", "exposure", "detector"), 

339 multiple=True, 

340 ) 

341 inputFluxes = cT.Input( 

342 name="crosstalkFluxes", 

343 doc="Fluxes of CT source pixels, for nonlinear fits.", 

344 storageClass="StructuredDataDict", 

345 dimensions=("instrument", "exposure", "detector"), 

346 multiple=True, 

347 ) 

348 camera = cT.PrerequisiteInput( 

349 name="camera", 

350 doc="Camera the input data comes from.", 

351 storageClass="Camera", 

352 dimensions=("instrument",), 

353 isCalibration=True, 

354 lookupFunction=lookupStaticCalibration, 

355 ) 

356 

357 outputCrosstalk = cT.Output( 

358 name="crosstalk", 

359 doc="Output proposed crosstalk calibration.", 

360 storageClass="CrosstalkCalib", 

361 dimensions=("instrument", "detector"), 

362 multiple=False, 

363 isCalibration=True, 

364 ) 

365 

366 def __init__(self, *, config=None): 

367 super().__init__(config=config) 

368 

369 if config.fluxOrder == 0: 

370 self.inputs.discard("inputFluxes") 

371 

372 

373class CrosstalkSolveConfig(pipeBase.PipelineTaskConfig, 

374 pipelineConnections=CrosstalkSolveConnections): 

375 """Configuration for the solving of crosstalk from pixel ratios. 

376 """ 

377 

378 rejIter = Field( 

379 dtype=int, 

380 default=3, 

381 doc="Number of rejection iterations for final coefficient calculation.", 

382 ) 

383 rejSigma = Field( 

384 dtype=float, 

385 default=2.0, 

386 doc="Rejection threshold (sigma) for final coefficient calculation.", 

387 ) 

388 fluxOrder = Field( 

389 dtype=int, 

390 default=0, 

391 doc="Polynomial order in source flux to fit crosstalk.", 

392 ) 

393 

394 rejectNegativeSolutions = Field( 

395 dtype=bool, 

396 default=True, 

397 doc="Should solutions with negative coefficients (which add flux to the target) be excluded?", 

398 ) 

399 

400 significanceLimit = Field( 

401 dtype=float, 

402 default=3.0, 

403 doc="Sigma significance level to use in marking a coefficient valid.", 

404 ) 

405 doSignificanceScaling = Field( 

406 dtype=bool, 

407 default=True, 

408 doc="Scale error by 1/sqrt(N) in calculating significant coefficients?", 

409 ) 

410 doFiltering = Field( 

411 dtype=bool, 

412 default=False, 

413 doc="Filter generated crosstalk to remove marginal measurements?", 

414 ) 

415 

416 

417class CrosstalkSolveTask(pipeBase.PipelineTask): 

418 """Task to solve crosstalk from pixel ratios. 

419 """ 

420 

421 ConfigClass = CrosstalkSolveConfig 

422 _DefaultName = 'cpCrosstalkSolve' 

423 

424 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

425 """Ensure that the input and output dimensions are passed along. 

426 

427 Parameters 

428 ---------- 

429 butlerQC : `lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

430 Butler to operate on. 

431 inputRefs : `lsst.pipe.base.connections.InputQuantizedConnection` 

432 Input data refs to load. 

433 ouptutRefs : `lsst.pipe.base.connections.OutputQuantizedConnection` 

434 Output data refs to persist. 

435 """ 

436 inputs = butlerQC.get(inputRefs) 

437 

438 # Use the dimensions to set calib/provenance information. 

439 inputs['inputDims'] = [exp.dataId.byName() for exp in inputRefs.inputRatios] 

440 inputs['outputDims'] = outputRefs.outputCrosstalk.dataId.byName() 

441 

442 outputs = self.run(**inputs) 

443 butlerQC.put(outputs, outputRefs) 

444 

445 def run(self, inputRatios, inputFluxes=None, camera=None, inputDims=None, outputDims=None): 

446 """Combine ratios to produce crosstalk coefficients. 

447 

448 Parameters 

449 ---------- 

450 inputRatios : `list` [`dict` [`dict` [`dict` [`dict` [`list`]]]]] 

451 A list of nested dictionaries of ratios indexed by target 

452 and source chip, then by target and source amplifier. 

453 inputFluxes : `list` [`dict` [`dict` [`list`]]] 

454 A list of nested dictionaries of source pixel fluxes, indexed 

455 by source chip and amplifier. 

456 camera : `lsst.afw.cameraGeom.Camera` 

457 Input camera. 

458 inputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

459 DataIds to use to construct provenance. 

460 outputDims : `list` [`lsst.daf.butler.DataCoordinate`] 

461 DataIds to use to populate the output calibration. 

462 

463 Returns 

464 ------- 

465 results : `lsst.pipe.base.Struct` 

466 The results struct containing: 

467 

468 ``outputCrosstalk`` 

469 Final crosstalk calibration 

470 (`lsst.ip.isr.CrosstalkCalib`). 

471 ``outputProvenance`` 

472 Provenance data for the new calibration 

473 (`lsst.ip.isr.IsrProvenance`). 

474 

475 Raises 

476 ------ 

477 RuntimeError 

478 Raised if the input data contains multiple target detectors. 

479 """ 

480 if outputDims: 

481 calibChip = outputDims['detector'] 

482 instrument = outputDims['instrument'] 

483 else: 

484 # calibChip needs to be set manually in Gen2. 

485 calibChip = None 

486 instrument = None 

487 

488 if camera and calibChip is not None: 

489 calibDetector = camera[calibChip] 

490 ordering = [amp.getName() for amp in calibDetector] 

491 else: 

492 calibDetector = None 

493 ordering = None 

494 

495 self.log.info("Combining measurements from %d ratios and %d fluxes", 

496 len(inputRatios), len(inputFluxes) if inputFluxes else 0) 

497 

498 if inputFluxes is None: 

499 inputFluxes = [None for exp in inputRatios] 

500 

501 combinedRatios = defaultdict(lambda: defaultdict(list)) 

502 combinedFluxes = defaultdict(lambda: defaultdict(list)) 

503 

504 for ratioDict, fluxDict in zip(inputRatios, inputFluxes): 

505 for targetChip in ratioDict: 

506 if calibChip and targetChip != calibChip and targetChip != calibDetector.getName(): 

507 raise RuntimeError(f"Target chip: {targetChip} does not match calibration dimension: " 

508 f"{calibChip}, {calibDetector.getName()}!") 

509 

510 sourceChip = targetChip 

511 if sourceChip in ratioDict[targetChip]: 

512 ratios = ratioDict[targetChip][sourceChip] 

513 

514 for targetAmp in ratios: 

515 for sourceAmp in ratios[targetAmp]: 

516 combinedRatios[targetAmp][sourceAmp].extend(ratios[targetAmp][sourceAmp]) 

517 if fluxDict: 

518 combinedFluxes[targetAmp][sourceAmp].extend(fluxDict[sourceChip][sourceAmp]) 

519 # TODO: DM-21904 

520 # Iterating over all other entries in 

521 # ratioDict[targetChip] will yield inter-chip terms. 

522 

523 for targetAmp in combinedRatios: 

524 for sourceAmp in combinedRatios[targetAmp]: 

525 self.log.info("Read %d pixels for %s -> %s", 

526 len(combinedRatios[targetAmp][sourceAmp]), 

527 sourceAmp, targetAmp) 

528 if len(combinedRatios[targetAmp][sourceAmp]) > 1: 

529 self.debugRatios('reduce', combinedRatios, targetAmp, sourceAmp) 

530 

531 if self.config.fluxOrder == 0: 

532 self.log.info("Fitting crosstalk coefficients.") 

533 

534 calib = self.measureCrosstalkCoefficients(combinedRatios, ordering, 

535 self.config.rejIter, self.config.rejSigma) 

536 else: 

537 raise NotImplementedError("Non-linear crosstalk terms are not yet supported.") 

538 

539 self.log.info("Number of valid coefficients: %d", np.sum(calib.coeffValid)) 

540 

541 if self.config.doFiltering: 

542 # This step will apply the calculated validity values to 

543 # censor poorly measured coefficients. 

544 self.log.info("Filtering measured crosstalk to remove invalid solutions.") 

545 calib = self.filterCrosstalkCalib(calib) 

546 

547 # Populate the remainder of the calibration information. 

548 calib.hasCrosstalk = True 

549 calib.interChip = {} 

550 

551 # calibChip is the detector dimension, which is the detector Id 

552 calib._detectorId = calibChip 

553 if calibDetector: 

554 calib._detectorName = calibDetector.getName() 

555 calib._detectorSerial = calibDetector.getSerial() 

556 

557 calib._instrument = instrument 

558 calib.updateMetadata(setCalibId=True, setDate=True) 

559 

560 # Make an IsrProvenance(). 

561 provenance = IsrProvenance(calibType="CROSSTALK") 

562 provenance._detectorName = calibChip 

563 if inputDims: 

564 provenance.fromDataIds(inputDims) 

565 provenance._instrument = instrument 

566 provenance.updateMetadata() 

567 

568 return pipeBase.Struct( 

569 outputCrosstalk=calib, 

570 outputProvenance=provenance, 

571 ) 

572 

573 def measureCrosstalkCoefficients(self, ratios, ordering, rejIter, rejSigma): 

574 """Measure crosstalk coefficients from the ratios. 

575 

576 Given a list of ratios for each target/source amp combination, 

577 we measure a sigma clipped mean and error. 

578 

579 The coefficient errors returned are the standard deviation of 

580 the final set of clipped input ratios. 

581 

582 Parameters 

583 ---------- 

584 ratios : `dict` [`dict` [`numpy.ndarray`]] 

585 Catalog of arrays of ratios. The ratio arrays are one-dimensional 

586 ordering : `list` [`str`] or None 

587 List to use as a mapping between amplifier names (the 

588 elements of the list) and their position in the output 

589 calibration (the matching index of the list). If no 

590 ordering is supplied, the order of the keys in the ratio 

591 catalog is used. 

592 rejIter : `int` 

593 Number of rejection iterations. 

594 rejSigma : `float` 

595 Rejection threshold (sigma). 

596 

597 Returns 

598 ------- 

599 calib : `lsst.ip.isr.CrosstalkCalib` 

600 The output crosstalk calibration. 

601 """ 

602 calib = CrosstalkCalib(nAmp=len(ratios)) 

603 

604 if ordering is None: 

605 ordering = list(ratios.keys()) 

606 

607 # Calibration stores coefficients as a numpy ndarray. 

608 for ss, tt in itertools.product(range(calib.nAmp), range(calib.nAmp)): 

609 if ss == tt: 

610 values = [0.0] 

611 else: 

612 # ratios is ratios[Target][Source] 

613 # use tt for Target, use ss for Source, to match ip_isr. 

614 values = np.array(ratios[ordering[tt]][ordering[ss]]) 

615 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

616 

617 # Sigma clip using the inter-quartile distance and a 

618 # normal distribution. 

619 if ss != tt: 

620 for rej in range(rejIter): 

621 if len(values) == 0: 

622 break 

623 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

624 sigma = 0.741*(hi - lo) 

625 good = np.abs(values - med) < rejSigma*sigma 

626 if good.sum() == len(good) or good.sum() == 0: 

627 break 

628 values = values[good] 

629 

630 # Crosstalk calib is property[Source][Target]. 

631 calib.coeffNum[ss][tt] = len(values) 

632 significanceThreshold = 0.0 

633 if len(values) == 0: 

634 self.log.warning("No values for matrix element %d,%d" % (ss, tt)) 

635 calib.coeffs[ss][tt] = np.nan 

636 calib.coeffErr[ss][tt] = np.nan 

637 calib.coeffValid[ss][tt] = False 

638 else: 

639 calib.coeffs[ss][tt] = np.mean(values) 

640 if self.config.rejectNegativeSolutions and calib.coeffs[ss][tt] < 0.0: 

641 calib.coeffs[ss][tt] = 0.0 

642 

643 if calib.coeffNum[ss][tt] == 1: 

644 calib.coeffErr[ss][tt] = np.nan 

645 calib.coeffValid[ss][tt] = False 

646 else: 

647 correctionFactor = sigmaClipCorrection(rejSigma) 

648 calib.coeffErr[ss][tt] = np.std(values) * correctionFactor 

649 

650 # Use sample stdev. 

651 significanceThreshold = self.config.significanceLimit * calib.coeffErr[ss][tt] 

652 if self.config.doSignificanceScaling is True: 

653 # Enabling this calculates the stdev of the mean. 

654 significanceThreshold /= np.sqrt(calib.coeffNum[ss][tt]) 

655 calib.coeffValid[ss][tt] = np.abs(calib.coeffs[ss][tt]) > significanceThreshold 

656 self.debugRatios('measure', ratios, ordering[ss], ordering[tt], 

657 calib.coeffs[ss][tt], calib.coeffValid[ss][tt]) 

658 self.log.info("Measured %s -> %s Coeff: %e Err: %e N: %d Valid: %s Limit: %e", 

659 ordering[ss], ordering[tt], calib.coeffs[ss][tt], calib.coeffErr[ss][tt], 

660 calib.coeffNum[ss][tt], calib.coeffValid[ss][tt], significanceThreshold) 

661 

662 return calib 

663 

664 @staticmethod 

665 def filterCrosstalkCalib(inCalib): 

666 """Apply valid constraints to the measured values. 

667 

668 Any measured coefficient that is determined to be invalid is 

669 set to zero, and has the error set to nan. The validation is 

670 determined by checking that the measured coefficient is larger 

671 than the calculated standard error of the mean. 

672 

673 Parameters 

674 ---------- 

675 inCalib : `lsst.ip.isr.CrosstalkCalib` 

676 Input calibration to filter. 

677 

678 Returns 

679 ------- 

680 outCalib : `lsst.ip.isr.CrosstalkCalib` 

681 Filtered calibration. 

682 """ 

683 outCalib = CrosstalkCalib() 

684 outCalib.nAmp = inCalib.nAmp 

685 

686 outCalib.coeffs = inCalib.coeffs 

687 outCalib.coeffs[~inCalib.coeffValid] = 0.0 

688 

689 outCalib.coeffErr = inCalib.coeffErr 

690 outCalib.coeffErr[~inCalib.coeffValid] = np.nan 

691 

692 outCalib.coeffNum = inCalib.coeffNum 

693 outCalib.coeffValid = inCalib.coeffValid 

694 

695 return outCalib 

696 

697 def debugRatios(self, stepname, ratios, i, j, coeff=0.0, valid=False): 

698 """Utility function to examine the final CT ratio set. 

699 

700 Parameters 

701 ---------- 

702 stepname : `str` 

703 State of processing to view. 

704 ratios : `dict` [`dict` [`numpy.ndarray`]] 

705 Array of measured CT ratios, indexed by source/victim 

706 amplifier. These arrays are one-dimensional. 

707 i : `str` 

708 Index of the target amplifier. 

709 j : `str` 

710 Index of the source amplifier. 

711 coeff : `float`, optional 

712 Coefficient calculated to plot along with the simple mean. 

713 valid : `bool`, optional 

714 Validity to be added to the plot title. 

715 """ 

716 frame = getDebugFrame(self._display, stepname) 

717 if frame: 

718 if i == j or ratios is None or len(ratios) < 1: 

719 pass 

720 

721 ratioList = ratios[i][j] 

722 if ratioList is None or len(ratioList) < 1: 

723 pass 

724 

725 mean = np.mean(ratioList) 

726 std = np.std(ratioList) 

727 import matplotlib.pyplot as plt 

728 figure = plt.figure(1) 

729 figure.clear() 

730 plt.hist(x=ratioList, bins=len(ratioList), 

731 cumulative=True, color='b', density=True, histtype='step') 

732 plt.xlabel("Measured pixel ratio") 

733 plt.ylabel(f"CDF: n={len(ratioList)}") 

734 plt.xlim(np.percentile(ratioList, [1.0, 99])) 

735 plt.axvline(x=mean, color="k") 

736 plt.axvline(x=coeff, color='g') 

737 plt.axvline(x=(std / np.sqrt(len(ratioList))), color='r') 

738 plt.axvline(x=-(std / np.sqrt(len(ratioList))), color='r') 

739 plt.title(f"(Source {j} -> Target {i}) mean: {mean:.2g} coeff: {coeff:.2g} valid: {valid}") 

740 figure.show() 

741 

742 prompt = "Press Enter to continue: " 

743 while True: 

744 ans = input(prompt).lower() 

745 if ans in ("", "c",): 

746 break 

747 elif ans in ("pdb", "p",): 

748 import pdb 

749 pdb.set_trace() 

750 plt.close()