Coverage for python/lsst/ip/isr/isrStatistics.py: 17%

249 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-01-27 10:05 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"] 

23 

24import numpy as np 

25 

26from scipy.signal.windows import hamming, hann, gaussian 

27from scipy.signal import butter, filtfilt 

28from scipy.stats import linregress 

29 

30import lsst.afw.math as afwMath 

31import lsst.afw.image as afwImage 

32import lsst.pipe.base as pipeBase 

33import lsst.pex.config as pexConfig 

34 

35from lsst.afw.cameraGeom import ReadoutCorner 

36 

37 

38class IsrStatisticsTaskConfig(pexConfig.Config): 

39 """Image statistics options. 

40 """ 

41 doCtiStatistics = pexConfig.Field( 

42 dtype=bool, 

43 doc="Measure CTI statistics from image and overscans?", 

44 default=False, 

45 ) 

46 doApplyGainsForCtiStatistics = pexConfig.Field( 

47 dtype=bool, 

48 doc="Apply gain to the overscan region when measuring CTI statistics?", 

49 default=True, 

50 ) 

51 

52 doBandingStatistics = pexConfig.Field( 

53 dtype=bool, 

54 doc="Measure image banding metric?", 

55 default=False, 

56 ) 

57 bandingKernelSize = pexConfig.Field( 57 ↛ exitline 57 didn't jump to the function exit

58 dtype=int, 

59 doc="Width of box for boxcar smoothing for banding metric.", 

60 default=3, 

61 check=lambda x: x == 0 or x % 2 != 0, 

62 ) 

63 bandingFractionLow = pexConfig.Field( 63 ↛ exitline 63 didn't jump to the function exit

64 dtype=float, 

65 doc="Fraction of values to exclude from low samples.", 

66 default=0.1, 

67 check=lambda x: x >= 0.0 and x <= 1.0 

68 ) 

69 bandingFractionHigh = pexConfig.Field( 69 ↛ exitline 69 didn't jump to the function exit

70 dtype=float, 

71 doc="Fraction of values to exclude from high samples.", 

72 default=0.9, 

73 check=lambda x: x >= 0.0 and x <= 1.0, 

74 ) 

75 bandingUseHalfDetector = pexConfig.Field( 

76 dtype=float, 

77 doc="Use only the first half set of amplifiers.", 

78 default=True, 

79 ) 

80 

81 doProjectionStatistics = pexConfig.Field( 

82 dtype=bool, 

83 doc="Measure projection metric?", 

84 default=False, 

85 ) 

86 projectionKernelSize = pexConfig.Field( 86 ↛ exitline 86 didn't jump to the function exit

87 dtype=int, 

88 doc="Width of box for boxcar smoothing of projections.", 

89 default=0, 

90 check=lambda x: x == 0 or x % 2 != 0, 

91 ) 

92 doProjectionFft = pexConfig.Field( 

93 dtype=bool, 

94 doc="Generate FFTs from the image projections?", 

95 default=False, 

96 ) 

97 projectionFftWindow = pexConfig.ChoiceField( 

98 dtype=str, 

99 doc="Type of windowing to use prior to calculating FFT.", 

100 default="HAMMING", 

101 allowed={ 

102 "HAMMING": "Hamming window.", 

103 "HANN": "Hann window.", 

104 "GAUSSIAN": "Gaussian window.", 

105 "NONE": "No window." 

106 } 

107 ) 

108 

109 doCopyCalibDistributionStatistics = pexConfig.Field( 

110 dtype=bool, 

111 doc="Copy calibration distribution statistics to output?", 

112 default=False, 

113 ) 

114 expectedDistributionLevels = pexConfig.ListField( 

115 dtype=float, 

116 doc="Percentile levels expected in the calibration header.", 

117 default=[0, 5, 16, 50, 84, 95, 100], 

118 ) 

119 

120 doBiasShiftStatistics = pexConfig.Field( 

121 dtype=bool, 

122 doc="Measure number of image shifts in overscan?", 

123 default=False, 

124 ) 

125 biasShiftFilterOrder = pexConfig.Field( 

126 dtype=int, 

127 doc="Filter order for Butterworth highpass filter.", 

128 default=5, 

129 ) 

130 biasShiftCutoff = pexConfig.Field( 

131 dtype=float, 

132 doc="Cutoff frequency for highpass filter.", 

133 default=1.0/15.0, 

134 ) 

135 biasShiftWindow = pexConfig.Field( 

136 dtype=int, 

137 doc="Filter window size in pixels for highpass filter.", 

138 default=30, 

139 ) 

140 biasShiftThreshold = pexConfig.Field( 

141 dtype=float, 

142 doc="S/N threshold for bias shift detection.", 

143 default=3.0, 

144 ) 

145 biasShiftRowSkip = pexConfig.Field( 

146 dtype=int, 

147 doc="Number of rows to skip for the bias shift detection.", 

148 default=30, 

149 ) 

150 biasShiftColumnSkip = pexConfig.Field( 

151 dtype=int, 

152 doc="Number of columns to skip when averaging the overscan region.", 

153 default=3, 

154 ) 

155 

156 doAmplifierCorrelationStatistics = pexConfig.Field( 

157 dtype=bool, 

158 doc="Measure amplifier correlations?", 

159 default=False, 

160 ) 

161 

162 stat = pexConfig.Field( 

163 dtype=str, 

164 default="MEANCLIP", 

165 doc="Statistic name to use to measure regions.", 

166 ) 

167 nSigmaClip = pexConfig.Field( 

168 dtype=float, 

169 default=3.0, 

170 doc="Clipping threshold for background", 

171 ) 

172 nIter = pexConfig.Field( 

173 dtype=int, 

174 default=3, 

175 doc="Clipping iterations for background", 

176 ) 

177 badMask = pexConfig.ListField( 

178 dtype=str, 

179 default=["BAD", "INTRP", "SAT"], 

180 doc="Mask planes to ignore when identifying source pixels." 

181 ) 

182 

183 

184class IsrStatisticsTask(pipeBase.Task): 

185 """Task to measure arbitrary statistics on ISR processed exposures. 

186 

187 The goal is to wrap a number of optional measurements that are 

188 useful for calibration production and detector stability. 

189 """ 

190 ConfigClass = IsrStatisticsTaskConfig 

191 _DefaultName = "isrStatistics" 

192 

193 def __init__(self, statControl=None, **kwargs): 

194 super().__init__(**kwargs) 

195 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter, 

196 afwImage.Mask.getPlaneBitMask(self.config.badMask)) 

197 self.statType = afwMath.stringToStatisticsProperty(self.config.stat) 

198 

199 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs): 

200 """Task to run arbitrary statistics. 

201 

202 The statistics should be measured by individual methods, and 

203 add to the dictionary in the return struct. 

204 

205 Parameters 

206 ---------- 

207 inputExp : `lsst.afw.image.Exposure` 

208 The exposure to measure. 

209 ptc : `lsst.ip.isr.PtcDataset`, optional 

210 A PTC object containing gains to use. 

211 overscanResults : `list` [`lsst.pipe.base.Struct`], optional 

212 List of overscan results. Expected fields are: 

213 

214 ``imageFit`` 

215 Value or fit subtracted from the amplifier image data 

216 (scalar or `lsst.afw.image.Image`). 

217 ``overscanFit`` 

218 Value or fit subtracted from the overscan image data 

219 (scalar or `lsst.afw.image.Image`). 

220 ``overscanImage`` 

221 Image of the overscan region with the overscan 

222 correction applied (`lsst.afw.image.Image`). This 

223 quantity is used to estimate the amplifier read noise 

224 empirically. 

225 

226 Returns 

227 ------- 

228 resultStruct : `lsst.pipe.base.Struct` 

229 Contains the measured statistics as a dict stored in a 

230 field named ``results``. 

231 

232 Raises 

233 ------ 

234 RuntimeError 

235 Raised if the amplifier gains could not be found. 

236 """ 

237 # Find gains. 

238 detector = inputExp.getDetector() 

239 if ptc is not None: 

240 gains = ptc.gain 

241 elif detector is not None: 

242 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()} 

243 else: 

244 raise RuntimeError("No source of gains provided.") 

245 

246 ctiResults = None 

247 if self.config.doCtiStatistics: 

248 ctiResults = self.measureCti(inputExp, overscanResults, gains) 

249 

250 bandingResults = None 

251 if self.config.doBandingStatistics: 

252 bandingResults = self.measureBanding(inputExp, overscanResults) 

253 

254 projectionResults = None 

255 if self.config.doProjectionStatistics: 

256 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults) 

257 

258 calibDistributionResults = None 

259 if self.config.doCopyCalibDistributionStatistics: 

260 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs) 

261 

262 biasShiftResults = None 

263 if self.config.doBiasShiftStatistics: 

264 biasShiftResults = self.measureBiasShifts(inputExp, overscanResults) 

265 

266 ampCorrelationResults = None 

267 if self.config.doAmplifierCorrelationStatistics: 

268 ampCorrelationResults = self.measureAmpCorrelations(inputExp, overscanResults) 

269 

270 return pipeBase.Struct( 

271 results={"CTI": ctiResults, 

272 "BANDING": bandingResults, 

273 "PROJECTION": projectionResults, 

274 "CALIBDIST": calibDistributionResults, 

275 "BIASSHIFT": biasShiftResults, 

276 "AMPCORR": ampCorrelationResults, 

277 }, 

278 ) 

279 

280 def measureCti(self, inputExp, overscans, gains): 

281 """Task to measure CTI statistics. 

282 

283 Parameters 

284 ---------- 

285 inputExp : `lsst.afw.image.Exposure` 

286 Exposure to measure. 

287 overscans : `list` [`lsst.pipe.base.Struct`] 

288 List of overscan results. Expected fields are: 

289 

290 ``imageFit`` 

291 Value or fit subtracted from the amplifier image data 

292 (scalar or `lsst.afw.image.Image`). 

293 ``overscanFit`` 

294 Value or fit subtracted from the overscan image data 

295 (scalar or `lsst.afw.image.Image`). 

296 ``overscanImage`` 

297 Image of the overscan region with the overscan 

298 correction applied (`lsst.afw.image.Image`). This 

299 quantity is used to estimate the amplifier read noise 

300 empirically. 

301 gains : `dict` [`str` `float`] 

302 Dictionary of per-amplifier gains, indexed by amplifier name. 

303 

304 Returns 

305 ------- 

306 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

307 Dictionary of measurements, keyed by amplifier name and 

308 statistics segment. 

309 """ 

310 outputStats = {} 

311 

312 detector = inputExp.getDetector() 

313 image = inputExp.image 

314 

315 # Ensure we have the same number of overscans as amplifiers. 

316 assert len(overscans) == len(detector.getAmplifiers()) 

317 

318 for ampIter, amp in enumerate(detector.getAmplifiers()): 

319 ampStats = {} 

320 gain = gains[amp.getName()] 

321 readoutCorner = amp.getReadoutCorner() 

322 # Full data region. 

323 dataRegion = image[amp.getBBox()] 

324 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType, 

325 self.statControl).getValue() 

326 

327 # First and last image columns. 

328 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0], 

329 self.statType, 

330 self.statControl).getValue() 

331 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1], 

332 self.statType, 

333 self.statControl).getValue() 

334 

335 # We want these relative to the readout corner. If that's 

336 # on the right side, we need to swap them. 

337 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

338 ampStats["FIRST_MEAN"] = pixelZ 

339 ampStats["LAST_MEAN"] = pixelA 

340 else: 

341 ampStats["FIRST_MEAN"] = pixelA 

342 ampStats["LAST_MEAN"] = pixelZ 

343 

344 # Measure the columns of the overscan. 

345 if overscans[ampIter] is None: 

346 # The amplifier is likely entirely bad, and needs to 

347 # be skipped. 

348 self.log.warning("No overscan information available for ISR statistics for amp %s.", 

349 amp.getName()) 

350 nCols = amp.getSerialOverscanBBox().getWidth() 

351 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan) 

352 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan) 

353 else: 

354 overscanImage = overscans[ampIter].overscanImage 

355 columns = [] 

356 values = [] 

357 for column in range(0, overscanImage.getWidth()): 

358 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column], 

359 self.statType, self.statControl).getValue() 

360 columns.append(column) 

361 if self.config.doApplyGainsForCtiStatistics: 

362 values.append(gain * osMean) 

363 else: 

364 values.append(osMean) 

365 

366 # We want these relative to the readout corner. If that's 

367 # on the right side, we need to swap them. 

368 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

369 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns)) 

370 ampStats["OVERSCAN_VALUES"] = list(reversed(values)) 

371 else: 

372 ampStats["OVERSCAN_COLUMNS"] = columns 

373 ampStats["OVERSCAN_VALUES"] = values 

374 

375 outputStats[amp.getName()] = ampStats 

376 

377 return outputStats 

378 

379 @staticmethod 

380 def makeKernel(kernelSize): 

381 """Make a boxcar smoothing kernel. 

382 

383 Parameters 

384 ---------- 

385 kernelSize : `int` 

386 Size of the kernel in pixels. 

387 

388 Returns 

389 ------- 

390 kernel : `np.array` 

391 Kernel for boxcar smoothing. 

392 """ 

393 if kernelSize > 0: 

394 kernel = np.full(kernelSize, 1.0 / kernelSize) 

395 else: 

396 kernel = np.array([1.0]) 

397 return kernel 

398 

399 def measureBanding(self, inputExp, overscans): 

400 """Task to measure banding statistics. 

401 

402 Parameters 

403 ---------- 

404 inputExp : `lsst.afw.image.Exposure` 

405 Exposure to measure. 

406 overscans : `list` [`lsst.pipe.base.Struct`] 

407 List of overscan results. Expected fields are: 

408 

409 ``imageFit`` 

410 Value or fit subtracted from the amplifier image data 

411 (scalar or `lsst.afw.image.Image`). 

412 ``overscanFit`` 

413 Value or fit subtracted from the overscan image data 

414 (scalar or `lsst.afw.image.Image`). 

415 ``overscanImage`` 

416 Image of the overscan region with the overscan 

417 correction applied (`lsst.afw.image.Image`). This 

418 quantity is used to estimate the amplifier read noise 

419 empirically. 

420 

421 Returns 

422 ------- 

423 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

424 Dictionary of measurements, keyed by amplifier name and 

425 statistics segment. 

426 """ 

427 outputStats = {} 

428 

429 detector = inputExp.getDetector() 

430 kernel = self.makeKernel(self.config.bandingKernelSize) 

431 

432 outputStats["AMP_BANDING"] = [] 

433 for amp, overscanData in zip(detector.getAmplifiers(), overscans): 

434 overscanFit = np.array(overscanData.overscanFit) 

435 overscanArray = overscanData.overscanImage.image.array 

436 rawOverscan = np.mean(overscanArray + overscanFit, axis=1) 

437 

438 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid") 

439 

440 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow, 

441 self.config.bandingFractionHigh]) 

442 outputStats["AMP_BANDING"].append(float(high - low)) 

443 

444 if self.config.bandingUseHalfDetector: 

445 fullLength = len(outputStats["AMP_BANDING"]) 

446 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2])) 

447 else: 

448 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"])) 

449 

450 return outputStats 

451 

452 def measureProjectionStatistics(self, inputExp, overscans): 

453 """Task to measure metrics from image slicing. 

454 

455 Parameters 

456 ---------- 

457 inputExp : `lsst.afw.image.Exposure` 

458 Exposure to measure. 

459 overscans : `list` [`lsst.pipe.base.Struct`] 

460 List of overscan results. Expected fields are: 

461 

462 ``imageFit`` 

463 Value or fit subtracted from the amplifier image data 

464 (scalar or `lsst.afw.image.Image`). 

465 ``overscanFit`` 

466 Value or fit subtracted from the overscan image data 

467 (scalar or `lsst.afw.image.Image`). 

468 ``overscanImage`` 

469 Image of the overscan region with the overscan 

470 correction applied (`lsst.afw.image.Image`). This 

471 quantity is used to estimate the amplifier read noise 

472 empirically. 

473 

474 Returns 

475 ------- 

476 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

477 Dictionary of measurements, keyed by amplifier name and 

478 statistics segment. 

479 """ 

480 outputStats = {} 

481 

482 detector = inputExp.getDetector() 

483 kernel = self.makeKernel(self.config.projectionKernelSize) 

484 

485 outputStats["AMP_VPROJECTION"] = {} 

486 outputStats["AMP_HPROJECTION"] = {} 

487 convolveMode = "valid" 

488 if self.config.doProjectionFft: 

489 outputStats["AMP_VFFT_REAL"] = {} 

490 outputStats["AMP_VFFT_IMAG"] = {} 

491 outputStats["AMP_HFFT_REAL"] = {} 

492 outputStats["AMP_HFFT_IMAG"] = {} 

493 convolveMode = "same" 

494 

495 for amp in detector.getAmplifiers(): 

496 ampArray = inputExp.image[amp.getBBox()].array 

497 

498 horizontalProjection = np.mean(ampArray, axis=0) 

499 verticalProjection = np.mean(ampArray, axis=1) 

500 

501 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode) 

502 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode) 

503 

504 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist() 

505 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist() 

506 

507 if self.config.doProjectionFft: 

508 horizontalWindow = np.ones_like(horizontalProjection) 

509 verticalWindow = np.ones_like(verticalProjection) 

510 if self.config.projectionFftWindow == "NONE": 

511 pass 

512 elif self.config.projectionFftWindow == "HAMMING": 

513 horizontalWindow = hamming(len(horizontalProjection)) 

514 verticalWindow = hamming(len(verticalProjection)) 

515 elif self.config.projectionFftWindow == "HANN": 

516 horizontalWindow = hann(len(horizontalProjection)) 

517 verticalWindow = hann(len(verticalProjection)) 

518 elif self.config.projectionFftWindow == "GAUSSIAN": 

519 horizontalWindow = gaussian(len(horizontalProjection)) 

520 verticalWindow = gaussian(len(verticalProjection)) 

521 else: 

522 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}") 

523 

524 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow)) 

525 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow)) 

526 

527 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist() 

528 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist() 

529 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist() 

530 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist() 

531 

532 return outputStats 

533 

534 def copyCalibDistributionStatistics(self, inputExp, **kwargs): 

535 """Copy calibration statistics for this exposure. 

536 

537 Parameters 

538 ---------- 

539 inputExp : `lsst.afw.image.Exposure` 

540 The exposure being processed. 

541 **kwargs : 

542 Keyword arguments with calibrations. 

543 

544 Returns 

545 ------- 

546 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

547 Dictionary of measurements, keyed by amplifier name and 

548 statistics segment. 

549 """ 

550 outputStats = {} 

551 

552 for amp in inputExp.getDetector(): 

553 ampStats = {} 

554 

555 for calibType in ("bias", "dark", "flat"): 

556 if kwargs.get(calibType, None) is not None: 

557 metadata = kwargs[calibType].getMetadata() 

558 for pct in self.config.expectedDistributionLevels: 

559 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT" 

560 ampStats[key] = metadata.get(key, np.nan) 

561 outputStats[amp.getName()] = ampStats 

562 return outputStats 

563 

564 def measureBiasShifts(self, inputExp, overscanResults): 

565 """Measure number of bias shifts from overscan data. 

566 

567 Parameters 

568 ---------- 

569 inputExp : `lsst.afw.image.Exposure` 

570 Exposure to measure. 

571 overscans : `list` [`lsst.pipe.base.Struct`] 

572 List of overscan results. Expected fields are: 

573 

574 ``imageFit`` 

575 Value or fit subtracted from the amplifier image data 

576 (scalar or `lsst.afw.image.Image`). 

577 ``overscanFit`` 

578 Value or fit subtracted from the overscan image data 

579 (scalar or `lsst.afw.image.Image`). 

580 ``overscanImage`` 

581 Image of the overscan region with the overscan 

582 correction applied (`lsst.afw.image.Image`). This 

583 quantity is used to estimate the amplifier read noise 

584 empirically. 

585 

586 Returns 

587 ------- 

588 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

589 Dictionary of measurements, keyed by amplifier name and 

590 statistics segment. 

591 

592 Notes 

593 ----- 

594 Based on eop_pipe implementation: 

595 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505 

596 """ 

597 outputStats = {} 

598 

599 detector = inputExp.getDetector() 

600 for amp, overscans in zip(detector, overscanResults): 

601 ampStats = {} 

602 # Add fit back to data 

603 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit 

604 

605 # Collapse array, skipping first three columns 

606 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1) 

607 

608 # Scan for shifts 

609 noise, shift_peaks = self._scan_for_shifts(rawOverscan) 

610 ampStats["LOCAL_NOISE"] = float(noise) 

611 ampStats["BIAS_SHIFTS"] = shift_peaks 

612 

613 outputStats[amp.getName()] = ampStats 

614 return outputStats 

615 

616 def _scan_for_shifts(self, overscanData): 

617 """Scan overscan data for shifts. 

618 

619 Parameters 

620 ---------- 

621 overscanData : `list` [`float`] 

622 Overscan data to search for shifts. 

623 

624 Returns 

625 ------- 

626 noise : `float` 

627 Noise estimated from Butterworth filtered overscan data. 

628 peaks : `list` [`float`, `float`, `int`, `int`] 

629 Shift peak information, containing the convolved peak 

630 value, the raw peak value, and the lower and upper bounds 

631 of the region checked. 

632 """ 

633 numerator, denominator = butter(self.config.biasShiftFilterOrder, 

634 self.config.biasShiftCutoff, 

635 btype="high", analog=False) 

636 noise = np.std(filtfilt(numerator, denominator, overscanData)) 

637 kernel = np.concatenate([np.arange(self.config.biasShiftWindow), 

638 np.arange(-self.config.biasShiftWindow + 1, 0)]) 

639 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow]) 

640 

641 convolved = np.convolve(overscanData, kernel, mode="valid") 

642 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow)) 

643 

644 shift_check = np.abs(convolved)/noise 

645 shift_mask = shift_check > self.config.biasShiftThreshold 

646 shift_mask[:self.config.biasShiftRowSkip] = False 

647 

648 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0), 

649 shift_mask.view(np.int8), 

650 np.int8(0)])).reshape(-1, 2) 

651 shift_peaks = [] 

652 for region in shift_regions: 

653 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0] 

654 if self._satisfies_flatness(region_peak, convolved[region_peak], overscanData): 

655 shift_peaks.append( 

656 [float(convolved[region_peak]), float(region_peak), 

657 int(region[0]), int(region[1])]) 

658 return noise, shift_peaks 

659 

660 def _satisfies_flatness(self, shiftRow, shiftPeak, overscanData): 

661 """Determine if a region is flat. 

662 

663 Parameters 

664 ---------- 

665 shiftRow : `int` 

666 Row with possible peak. 

667 shiftPeak : `float` 

668 Value at the possible peak. 

669 overscanData : `list` [`float`] 

670 Overscan data used to fit around the possible peak. 

671 

672 Returns 

673 ------- 

674 isFlat : `bool` 

675 Indicates if the region is flat, and so the peak is valid. 

676 """ 

677 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow) 

678 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow) 

679 

680 preFit = linregress(prerange, overscanData[prerange]) 

681 postFit = linregress(postrange, overscanData[postrange]) 

682 

683 if shiftPeak > 0: 

684 preTrend = (2*preFit[0]*len(prerange) < shiftPeak) 

685 postTrend = (2*postFit[0]*len(postrange) < shiftPeak) 

686 else: 

687 preTrend = (2*preFit[0]*len(prerange) > shiftPeak) 

688 postTrend = (2*postFit[0]*len(postrange) > shiftPeak) 

689 

690 return (preTrend and postTrend) 

691 

692 def measureAmpCorrelations(self, inputExp, overscanResults): 

693 """Measure correlations between amplifier segments. 

694 

695 Parameters 

696 ---------- 

697 inputExp : `lsst.afw.image.Exposure` 

698 Exposure to measure. 

699 overscans : `list` [`lsst.pipe.base.Struct`] 

700 List of overscan results. Expected fields are: 

701 

702 ``imageFit`` 

703 Value or fit subtracted from the amplifier image data 

704 (scalar or `lsst.afw.image.Image`). 

705 ``overscanFit`` 

706 Value or fit subtracted from the overscan image data 

707 (scalar or `lsst.afw.image.Image`). 

708 ``overscanImage`` 

709 Image of the overscan region with the overscan 

710 correction applied (`lsst.afw.image.Image`). This 

711 quantity is used to estimate the amplifier read noise 

712 empirically. 

713 

714 Returns 

715 ------- 

716 outputStats : `dict` [`str`, [`dict` [`str`,`float`]] 

717 Dictionary of measurements, keyed by amplifier name and 

718 statistics segment. 

719 

720 Notes 

721 ----- 

722 Based on eo_pipe implementation: 

723 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505 

724 """ 

725 outputStats = {} 

726 

727 detector = inputExp.getDetector() 

728 

729 serialOSCorr = np.empty((len(detector), len(detector))) 

730 imageCorr = np.empty((len(detector), len(detector))) 

731 for ampId, overscan in enumerate(overscanResults): 

732 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit 

733 rawOverscan = rawOverscan.ravel() 

734 

735 ampImage = inputExp[detector[ampId].getBBox()] 

736 ampImage = ampImage.image.array.ravel() 

737 

738 for ampId2, overscan2 in enumerate(overscanResults): 

739 

740 if ampId2 == ampId: 

741 serialOSCorr[ampId, ampId2] = 1.0 

742 imageCorr[ampId, ampId2] = 1.0 

743 else: 

744 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit 

745 rawOverscan2 = rawOverscan2.ravel() 

746 

747 serialOSCorr[ampId, ampId2] = np.corrcoef(rawOverscan, rawOverscan2)[0, 1] 

748 

749 ampImage2 = inputExp[detector[ampId2].getBBox()] 

750 ampImage2 = ampImage2.image.array.ravel() 

751 

752 imageCorr[ampId, ampId2] = np.corrcoef(ampImage, ampImage2)[0, 1] 

753 

754 outputStats["OVERSCAN_CORR"] = serialOSCorr.tolist() 

755 outputStats["IMAGE_CORR"] = imageCorr.tolist() 

756 

757 return outputStats