Coverage for python/lsst/ip/isr/isrStatistics.py: 17%

251 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-03 11:38 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"] 

23 

24import numpy as np 

25 

26from scipy.signal.windows import hamming, hann, gaussian 

27from scipy.signal import butter, filtfilt 

28from scipy.stats import linregress 

29 

30import lsst.afw.math as afwMath 

31import lsst.afw.image as afwImage 

32import lsst.pipe.base as pipeBase 

33import lsst.pex.config as pexConfig 

34 

35from lsst.afw.cameraGeom import ReadoutCorner 

36 

37 

38class IsrStatisticsTaskConfig(pexConfig.Config): 

39 """Image statistics options. 

40 """ 

41 doCtiStatistics = pexConfig.Field( 

42 dtype=bool, 

43 doc="Measure CTI statistics from image and overscans?", 

44 default=False, 

45 ) 

46 doApplyGainsForCtiStatistics = pexConfig.Field( 

47 dtype=bool, 

48 doc="Apply gain to the overscan region when measuring CTI statistics?", 

49 default=True, 

50 ) 

51 

52 doBandingStatistics = pexConfig.Field( 

53 dtype=bool, 

54 doc="Measure image banding metric?", 

55 default=False, 

56 ) 

57 bandingKernelSize = pexConfig.Field( 57 ↛ exitline 57 didn't jump to the function exit

58 dtype=int, 

59 doc="Width of box for boxcar smoothing for banding metric.", 

60 default=3, 

61 check=lambda x: x == 0 or x % 2 != 0, 

62 ) 

63 bandingFractionLow = pexConfig.Field( 63 ↛ exitline 63 didn't jump to the function exit

64 dtype=float, 

65 doc="Fraction of values to exclude from low samples.", 

66 default=0.1, 

67 check=lambda x: x >= 0.0 and x <= 1.0 

68 ) 

69 bandingFractionHigh = pexConfig.Field( 69 ↛ exitline 69 didn't jump to the function exit

70 dtype=float, 

71 doc="Fraction of values to exclude from high samples.", 

72 default=0.9, 

73 check=lambda x: x >= 0.0 and x <= 1.0, 

74 ) 

75 bandingUseHalfDetector = pexConfig.Field( 

76 dtype=float, 

77 doc="Use only the first half set of amplifiers.", 

78 default=True, 

79 ) 

80 

81 doProjectionStatistics = pexConfig.Field( 

82 dtype=bool, 

83 doc="Measure projection metric?", 

84 default=False, 

85 ) 

86 projectionKernelSize = pexConfig.Field( 86 ↛ exitline 86 didn't jump to the function exit

87 dtype=int, 

88 doc="Width of box for boxcar smoothing of projections.", 

89 default=0, 

90 check=lambda x: x == 0 or x % 2 != 0, 

91 ) 

92 doProjectionFft = pexConfig.Field( 

93 dtype=bool, 

94 doc="Generate FFTs from the image projections?", 

95 default=False, 

96 ) 

97 projectionFftWindow = pexConfig.ChoiceField( 

98 dtype=str, 

99 doc="Type of windowing to use prior to calculating FFT.", 

100 default="HAMMING", 

101 allowed={ 

102 "HAMMING": "Hamming window.", 

103 "HANN": "Hann window.", 

104 "GAUSSIAN": "Gaussian window.", 

105 "NONE": "No window." 

106 } 

107 ) 

108 

109 doCopyCalibDistributionStatistics = pexConfig.Field( 

110 dtype=bool, 

111 doc="Copy calibration distribution statistics to output?", 

112 default=False, 

113 ) 

114 expectedDistributionLevels = pexConfig.ListField( 

115 dtype=float, 

116 doc="Percentile levels expected in the calibration header.", 

117 default=[0, 5, 16, 50, 84, 95, 100], 

118 ) 

119 

120 doBiasShiftStatistics = pexConfig.Field( 

121 dtype=bool, 

122 doc="Measure number of image shifts in overscan?", 

123 default=False, 

124 ) 

125 biasShiftFilterOrder = pexConfig.Field( 

126 dtype=int, 

127 doc="Filter order for Butterworth highpass filter.", 

128 default=5, 

129 ) 

130 biasShiftCutoff = pexConfig.Field( 

131 dtype=float, 

132 doc="Cutoff frequency for highpass filter.", 

133 default=1.0/15.0, 

134 ) 

135 biasShiftWindow = pexConfig.Field( 

136 dtype=int, 

137 doc="Filter window size in pixels for highpass filter.", 

138 default=30, 

139 ) 

140 biasShiftThreshold = pexConfig.Field( 

141 dtype=float, 

142 doc="S/N threshold for bias shift detection.", 

143 default=3.0, 

144 ) 

145 biasShiftRowSkip = pexConfig.Field( 

146 dtype=int, 

147 doc="Number of rows to skip for the bias shift detection.", 

148 default=30, 

149 ) 

150 biasShiftColumnSkip = pexConfig.Field( 

151 dtype=int, 

152 doc="Number of columns to skip when averaging the overscan region.", 

153 default=3, 

154 ) 

155 

156 doAmplifierCorrelationStatistics = pexConfig.Field( 

157 dtype=bool, 

158 doc="Measure amplifier correlations?", 

159 default=False, 

160 ) 

161 

162 stat = pexConfig.Field( 

163 dtype=str, 

164 default="MEANCLIP", 

165 doc="Statistic name to use to measure regions.", 

166 ) 

167 nSigmaClip = pexConfig.Field( 

168 dtype=float, 

169 default=3.0, 

170 doc="Clipping threshold for background", 

171 ) 

172 nIter = pexConfig.Field( 

173 dtype=int, 

174 default=3, 

175 doc="Clipping iterations for background", 

176 ) 

177 badMask = pexConfig.ListField( 

178 dtype=str, 

179 default=["BAD", "INTRP", "SAT"], 

180 doc="Mask planes to ignore when identifying source pixels." 

181 ) 

182 

183 

184class IsrStatisticsTask(pipeBase.Task): 

185 """Task to measure arbitrary statistics on ISR processed exposures. 

186 

187 The goal is to wrap a number of optional measurements that are 

188 useful for calibration production and detector stability. 

189 """ 

190 ConfigClass = IsrStatisticsTaskConfig 

191 _DefaultName = "isrStatistics" 

192 

193 def __init__(self, statControl=None, **kwargs): 

194 super().__init__(**kwargs) 

195 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter, 

196 afwImage.Mask.getPlaneBitMask(self.config.badMask)) 

197 self.statType = afwMath.stringToStatisticsProperty(self.config.stat) 

198 

199 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs): 

200 """Task to run arbitrary statistics. 

201 

202 The statistics should be measured by individual methods, and 

203 add to the dictionary in the return struct. 

204 

205 Parameters 

206 ---------- 

207 inputExp : `lsst.afw.image.Exposure` 

208 The exposure to measure. 

209 ptc : `lsst.ip.isr.PtcDataset`, optional 

210 A PTC object containing gains to use. 

211 overscanResults : `list` [`lsst.pipe.base.Struct`], optional 

212 List of overscan results. Expected fields are: 

213 

214 ``imageFit`` 

215 Value or fit subtracted from the amplifier image data 

216 (scalar or `lsst.afw.image.Image`). 

217 ``overscanFit`` 

218 Value or fit subtracted from the overscan image data 

219 (scalar or `lsst.afw.image.Image`). 

220 ``overscanImage`` 

221 Image of the overscan region with the overscan 

222 correction applied (`lsst.afw.image.Image`). This 

223 quantity is used to estimate the amplifier read noise 

224 empirically. 

225 

226 Returns 

227 ------- 

228 resultStruct : `lsst.pipe.base.Struct` 

229 Contains the measured statistics as a dict stored in a 

230 field named ``results``. 

231 

232 Raises 

233 ------ 

234 RuntimeError 

235 Raised if the amplifier gains could not be found. 

236 """ 

237 # Find gains. 

238 detector = inputExp.getDetector() 

239 if ptc is not None: 

240 gains = ptc.gain 

241 elif detector is not None: 

242 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()} 

243 else: 

244 raise RuntimeError("No source of gains provided.") 

245 

246 ctiResults = None 

247 if self.config.doCtiStatistics: 

248 ctiResults = self.measureCti(inputExp, overscanResults, gains) 

249 

250 bandingResults = None 

251 if self.config.doBandingStatistics: 

252 bandingResults = self.measureBanding(inputExp, overscanResults) 

253 

254 projectionResults = None 

255 if self.config.doProjectionStatistics: 

256 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults) 

257 

258 calibDistributionResults = None 

259 if self.config.doCopyCalibDistributionStatistics: 

260 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs) 

261 

262 biasShiftResults = None 

263 if self.config.doBiasShiftStatistics: 

264 biasShiftResults = self.measureBiasShifts(inputExp, overscanResults) 

265 

266 ampCorrelationResults = None 

267 if self.config.doAmplifierCorrelationStatistics: 

268 ampCorrelationResults = self.measureAmpCorrelations(inputExp, overscanResults) 

269 

270 mjd = inputExp.getMetadata().get("MJD", None) 

271 

272 return pipeBase.Struct( 

273 results={"CTI": ctiResults, 

274 "BANDING": bandingResults, 

275 "PROJECTION": projectionResults, 

276 "CALIBDIST": calibDistributionResults, 

277 "BIASSHIFT": biasShiftResults, 

278 "AMPCORR": ampCorrelationResults, 

279 "MJD": mjd, 

280 }, 

281 ) 

282 

283 def measureCti(self, inputExp, overscans, gains): 

284 """Task to measure CTI statistics. 

285 

286 Parameters 

287 ---------- 

288 inputExp : `lsst.afw.image.Exposure` 

289 Exposure to measure. 

290 overscans : `list` [`lsst.pipe.base.Struct`] 

291 List of overscan results. Expected fields are: 

292 

293 ``imageFit`` 

294 Value or fit subtracted from the amplifier image data 

295 (scalar or `lsst.afw.image.Image`). 

296 ``overscanFit`` 

297 Value or fit subtracted from the overscan image data 

298 (scalar or `lsst.afw.image.Image`). 

299 ``overscanImage`` 

300 Image of the overscan region with the overscan 

301 correction applied (`lsst.afw.image.Image`). This 

302 quantity is used to estimate the amplifier read noise 

303 empirically. 

304 gains : `dict` [`str` `float`] 

305 Dictionary of per-amplifier gains, indexed by amplifier name. 

306 

307 Returns 

308 ------- 

309 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

310 Dictionary of measurements, keyed by amplifier name and 

311 statistics segment. 

312 """ 

313 outputStats = {} 

314 

315 detector = inputExp.getDetector() 

316 image = inputExp.image 

317 

318 # Ensure we have the same number of overscans as amplifiers. 

319 assert len(overscans) == len(detector.getAmplifiers()) 

320 

321 for ampIter, amp in enumerate(detector.getAmplifiers()): 

322 ampStats = {} 

323 gain = gains[amp.getName()] 

324 readoutCorner = amp.getReadoutCorner() 

325 # Full data region. 

326 dataRegion = image[amp.getBBox()] 

327 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType, 

328 self.statControl).getValue() 

329 

330 # First and last image columns. 

331 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0], 

332 self.statType, 

333 self.statControl).getValue() 

334 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1], 

335 self.statType, 

336 self.statControl).getValue() 

337 

338 # We want these relative to the readout corner. If that's 

339 # on the right side, we need to swap them. 

340 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

341 ampStats["FIRST_MEAN"] = pixelZ 

342 ampStats["LAST_MEAN"] = pixelA 

343 else: 

344 ampStats["FIRST_MEAN"] = pixelA 

345 ampStats["LAST_MEAN"] = pixelZ 

346 

347 # Measure the columns of the overscan. 

348 if overscans[ampIter] is None: 

349 # The amplifier is likely entirely bad, and needs to 

350 # be skipped. 

351 self.log.warning("No overscan information available for ISR statistics for amp %s.", 

352 amp.getName()) 

353 nCols = amp.getRawSerialOverscanBBox().getWidth() 

354 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan) 

355 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan) 

356 else: 

357 overscanImage = overscans[ampIter].overscanImage 

358 columns = [] 

359 values = [] 

360 for column in range(0, overscanImage.getWidth()): 

361 # If overscan.doParallelOverscan=True, the overscanImage 

362 # will contain both the serial and parallel overscan 

363 # regions. 

364 # Only the serial overscan correction is implemented, 

365 # so we must select only the serial overscan rows 

366 # for a given column. 

367 nRows = amp.getRawSerialOverscanBBox().getHeight() 

368 osMean = afwMath.makeStatistics(overscanImage.image.array[:nRows, column], 

369 self.statType, self.statControl).getValue() 

370 columns.append(column) 

371 if self.config.doApplyGainsForCtiStatistics: 

372 values.append(gain * osMean) 

373 else: 

374 values.append(osMean) 

375 

376 # We want these relative to the readout corner. If that's 

377 # on the right side, we need to swap them. 

378 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

379 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns)) 

380 ampStats["OVERSCAN_VALUES"] = list(reversed(values)) 

381 else: 

382 ampStats["OVERSCAN_COLUMNS"] = columns 

383 ampStats["OVERSCAN_VALUES"] = values 

384 

385 outputStats[amp.getName()] = ampStats 

386 

387 return outputStats 

388 

389 @staticmethod 

390 def makeKernel(kernelSize): 

391 """Make a boxcar smoothing kernel. 

392 

393 Parameters 

394 ---------- 

395 kernelSize : `int` 

396 Size of the kernel in pixels. 

397 

398 Returns 

399 ------- 

400 kernel : `np.array` 

401 Kernel for boxcar smoothing. 

402 """ 

403 if kernelSize > 0: 

404 kernel = np.full(kernelSize, 1.0 / kernelSize) 

405 else: 

406 kernel = np.array([1.0]) 

407 return kernel 

408 

409 def measureBanding(self, inputExp, overscans): 

410 """Task to measure banding statistics. 

411 

412 Parameters 

413 ---------- 

414 inputExp : `lsst.afw.image.Exposure` 

415 Exposure to measure. 

416 overscans : `list` [`lsst.pipe.base.Struct`] 

417 List of overscan results. Expected fields are: 

418 

419 ``imageFit`` 

420 Value or fit subtracted from the amplifier image data 

421 (scalar or `lsst.afw.image.Image`). 

422 ``overscanFit`` 

423 Value or fit subtracted from the overscan image data 

424 (scalar or `lsst.afw.image.Image`). 

425 ``overscanImage`` 

426 Image of the overscan region with the overscan 

427 correction applied (`lsst.afw.image.Image`). This 

428 quantity is used to estimate the amplifier read noise 

429 empirically. 

430 

431 Returns 

432 ------- 

433 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

434 Dictionary of measurements, keyed by amplifier name and 

435 statistics segment. 

436 """ 

437 outputStats = {} 

438 

439 detector = inputExp.getDetector() 

440 kernel = self.makeKernel(self.config.bandingKernelSize) 

441 

442 outputStats["AMP_BANDING"] = [] 

443 for amp, overscanData in zip(detector.getAmplifiers(), overscans): 

444 overscanFit = np.array(overscanData.overscanFit) 

445 overscanArray = overscanData.overscanImage.image.array 

446 rawOverscan = np.mean(overscanArray + overscanFit, axis=1) 

447 

448 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid") 

449 

450 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow, 

451 self.config.bandingFractionHigh]) 

452 outputStats["AMP_BANDING"].append(float(high - low)) 

453 

454 if self.config.bandingUseHalfDetector: 

455 fullLength = len(outputStats["AMP_BANDING"]) 

456 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2])) 

457 else: 

458 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"])) 

459 

460 return outputStats 

461 

462 def measureProjectionStatistics(self, inputExp, overscans): 

463 """Task to measure metrics from image slicing. 

464 

465 Parameters 

466 ---------- 

467 inputExp : `lsst.afw.image.Exposure` 

468 Exposure to measure. 

469 overscans : `list` [`lsst.pipe.base.Struct`] 

470 List of overscan results. Expected fields are: 

471 

472 ``imageFit`` 

473 Value or fit subtracted from the amplifier image data 

474 (scalar or `lsst.afw.image.Image`). 

475 ``overscanFit`` 

476 Value or fit subtracted from the overscan image data 

477 (scalar or `lsst.afw.image.Image`). 

478 ``overscanImage`` 

479 Image of the overscan region with the overscan 

480 correction applied (`lsst.afw.image.Image`). This 

481 quantity is used to estimate the amplifier read noise 

482 empirically. 

483 

484 Returns 

485 ------- 

486 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

487 Dictionary of measurements, keyed by amplifier name and 

488 statistics segment. 

489 """ 

490 outputStats = {} 

491 

492 detector = inputExp.getDetector() 

493 kernel = self.makeKernel(self.config.projectionKernelSize) 

494 

495 outputStats["AMP_VPROJECTION"] = {} 

496 outputStats["AMP_HPROJECTION"] = {} 

497 convolveMode = "valid" 

498 if self.config.doProjectionFft: 

499 outputStats["AMP_VFFT_REAL"] = {} 

500 outputStats["AMP_VFFT_IMAG"] = {} 

501 outputStats["AMP_HFFT_REAL"] = {} 

502 outputStats["AMP_HFFT_IMAG"] = {} 

503 convolveMode = "same" 

504 

505 for amp in detector.getAmplifiers(): 

506 ampArray = inputExp.image[amp.getBBox()].array 

507 

508 horizontalProjection = np.mean(ampArray, axis=0) 

509 verticalProjection = np.mean(ampArray, axis=1) 

510 

511 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode) 

512 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode) 

513 

514 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist() 

515 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist() 

516 

517 if self.config.doProjectionFft: 

518 horizontalWindow = np.ones_like(horizontalProjection) 

519 verticalWindow = np.ones_like(verticalProjection) 

520 if self.config.projectionFftWindow == "NONE": 

521 pass 

522 elif self.config.projectionFftWindow == "HAMMING": 

523 horizontalWindow = hamming(len(horizontalProjection)) 

524 verticalWindow = hamming(len(verticalProjection)) 

525 elif self.config.projectionFftWindow == "HANN": 

526 horizontalWindow = hann(len(horizontalProjection)) 

527 verticalWindow = hann(len(verticalProjection)) 

528 elif self.config.projectionFftWindow == "GAUSSIAN": 

529 horizontalWindow = gaussian(len(horizontalProjection)) 

530 verticalWindow = gaussian(len(verticalProjection)) 

531 else: 

532 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}") 

533 

534 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow)) 

535 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow)) 

536 

537 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist() 

538 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist() 

539 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist() 

540 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist() 

541 

542 return outputStats 

543 

544 def copyCalibDistributionStatistics(self, inputExp, **kwargs): 

545 """Copy calibration statistics for this exposure. 

546 

547 Parameters 

548 ---------- 

549 inputExp : `lsst.afw.image.Exposure` 

550 The exposure being processed. 

551 **kwargs : 

552 Keyword arguments with calibrations. 

553 

554 Returns 

555 ------- 

556 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

557 Dictionary of measurements, keyed by amplifier name and 

558 statistics segment. 

559 """ 

560 outputStats = {} 

561 

562 for amp in inputExp.getDetector(): 

563 ampStats = {} 

564 

565 for calibType in ("bias", "dark", "flat"): 

566 if kwargs.get(calibType, None) is not None: 

567 metadata = kwargs[calibType].getMetadata() 

568 for pct in self.config.expectedDistributionLevels: 

569 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT" 

570 ampStats[key] = metadata.get(key, np.nan) 

571 outputStats[amp.getName()] = ampStats 

572 return outputStats 

573 

574 def measureBiasShifts(self, inputExp, overscanResults): 

575 """Measure number of bias shifts from overscan data. 

576 

577 Parameters 

578 ---------- 

579 inputExp : `lsst.afw.image.Exposure` 

580 Exposure to measure. 

581 overscans : `list` [`lsst.pipe.base.Struct`] 

582 List of overscan results. Expected fields are: 

583 

584 ``imageFit`` 

585 Value or fit subtracted from the amplifier image data 

586 (scalar or `lsst.afw.image.Image`). 

587 ``overscanFit`` 

588 Value or fit subtracted from the overscan image data 

589 (scalar or `lsst.afw.image.Image`). 

590 ``overscanImage`` 

591 Image of the overscan region with the overscan 

592 correction applied (`lsst.afw.image.Image`). This 

593 quantity is used to estimate the amplifier read noise 

594 empirically. 

595 

596 Returns 

597 ------- 

598 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

599 Dictionary of measurements, keyed by amplifier name and 

600 statistics segment. 

601 

602 Notes 

603 ----- 

604 Based on eop_pipe implementation: 

605 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/biasShiftsTask.py # noqa: E501 W505 

606 """ 

607 outputStats = {} 

608 

609 detector = inputExp.getDetector() 

610 for amp, overscans in zip(detector, overscanResults): 

611 ampStats = {} 

612 # Add fit back to data 

613 rawOverscan = overscans.overscanImage.image.array + overscans.overscanFit 

614 

615 # Collapse array, skipping first three columns 

616 rawOverscan = np.mean(rawOverscan[:, self.config.biasShiftColumnSkip:], axis=1) 

617 

618 # Scan for shifts 

619 noise, shift_peaks = self._scan_for_shifts(rawOverscan) 

620 ampStats["LOCAL_NOISE"] = float(noise) 

621 ampStats["BIAS_SHIFTS"] = shift_peaks 

622 

623 outputStats[amp.getName()] = ampStats 

624 return outputStats 

625 

626 def _scan_for_shifts(self, overscanData): 

627 """Scan overscan data for shifts. 

628 

629 Parameters 

630 ---------- 

631 overscanData : `list` [`float`] 

632 Overscan data to search for shifts. 

633 

634 Returns 

635 ------- 

636 noise : `float` 

637 Noise estimated from Butterworth filtered overscan data. 

638 peaks : `list` [`float`, `float`, `int`, `int`] 

639 Shift peak information, containing the convolved peak 

640 value, the raw peak value, and the lower and upper bounds 

641 of the region checked. 

642 """ 

643 numerator, denominator = butter(self.config.biasShiftFilterOrder, 

644 self.config.biasShiftCutoff, 

645 btype="high", analog=False) 

646 noise = np.std(filtfilt(numerator, denominator, overscanData)) 

647 kernel = np.concatenate([np.arange(self.config.biasShiftWindow), 

648 np.arange(-self.config.biasShiftWindow + 1, 0)]) 

649 kernel = kernel/np.sum(kernel[:self.config.biasShiftWindow]) 

650 

651 convolved = np.convolve(overscanData, kernel, mode="valid") 

652 convolved = np.pad(convolved, (self.config.biasShiftWindow - 1, self.config.biasShiftWindow)) 

653 

654 shift_check = np.abs(convolved)/noise 

655 shift_mask = shift_check > self.config.biasShiftThreshold 

656 shift_mask[:self.config.biasShiftRowSkip] = False 

657 

658 shift_regions = np.flatnonzero(np.diff(np.r_[np.int8(0), 

659 shift_mask.view(np.int8), 

660 np.int8(0)])).reshape(-1, 2) 

661 shift_peaks = [] 

662 for region in shift_regions: 

663 region_peak = np.argmax(shift_check[region[0]:region[1]]) + region[0] 

664 if self._satisfies_flatness(region_peak, convolved[region_peak], overscanData): 

665 shift_peaks.append( 

666 [float(convolved[region_peak]), float(region_peak), 

667 int(region[0]), int(region[1])]) 

668 return noise, shift_peaks 

669 

670 def _satisfies_flatness(self, shiftRow, shiftPeak, overscanData): 

671 """Determine if a region is flat. 

672 

673 Parameters 

674 ---------- 

675 shiftRow : `int` 

676 Row with possible peak. 

677 shiftPeak : `float` 

678 Value at the possible peak. 

679 overscanData : `list` [`float`] 

680 Overscan data used to fit around the possible peak. 

681 

682 Returns 

683 ------- 

684 isFlat : `bool` 

685 Indicates if the region is flat, and so the peak is valid. 

686 """ 

687 prerange = np.arange(shiftRow - self.config.biasShiftWindow, shiftRow) 

688 postrange = np.arange(shiftRow, shiftRow + self.config.biasShiftWindow) 

689 

690 preFit = linregress(prerange, overscanData[prerange]) 

691 postFit = linregress(postrange, overscanData[postrange]) 

692 

693 if shiftPeak > 0: 

694 preTrend = (2*preFit[0]*len(prerange) < shiftPeak) 

695 postTrend = (2*postFit[0]*len(postrange) < shiftPeak) 

696 else: 

697 preTrend = (2*preFit[0]*len(prerange) > shiftPeak) 

698 postTrend = (2*postFit[0]*len(postrange) > shiftPeak) 

699 

700 return (preTrend and postTrend) 

701 

702 def measureAmpCorrelations(self, inputExp, overscanResults): 

703 """Measure correlations between amplifier segments. 

704 

705 Parameters 

706 ---------- 

707 inputExp : `lsst.afw.image.Exposure` 

708 Exposure to measure. 

709 overscans : `list` [`lsst.pipe.base.Struct`] 

710 List of overscan results. Expected fields are: 

711 

712 ``imageFit`` 

713 Value or fit subtracted from the amplifier image data 

714 (scalar or `lsst.afw.image.Image`). 

715 ``overscanFit`` 

716 Value or fit subtracted from the overscan image data 

717 (scalar or `lsst.afw.image.Image`). 

718 ``overscanImage`` 

719 Image of the overscan region with the overscan 

720 correction applied (`lsst.afw.image.Image`). This 

721 quantity is used to estimate the amplifier read noise 

722 empirically. 

723 

724 Returns 

725 ------- 

726 outputStats : `dict` [`str`, [`dict` [`str`,`float`]] 

727 Dictionary of measurements, keyed by amplifier name and 

728 statistics segment. 

729 

730 Notes 

731 ----- 

732 Based on eo_pipe implementation: 

733 https://github.com/lsst-camera-dh/eo_pipe/blob/main/python/lsst/eo/pipe/raft_level_correlations.py # noqa: E501 W505 

734 """ 

735 outputStats = {} 

736 

737 detector = inputExp.getDetector() 

738 

739 serialOSCorr = np.empty((len(detector), len(detector))) 

740 imageCorr = np.empty((len(detector), len(detector))) 

741 for ampId, overscan in enumerate(overscanResults): 

742 rawOverscan = overscan.overscanImage.image.array + overscan.overscanFit 

743 rawOverscan = rawOverscan.ravel() 

744 

745 ampImage = inputExp[detector[ampId].getBBox()] 

746 ampImage = ampImage.image.array.ravel() 

747 

748 for ampId2, overscan2 in enumerate(overscanResults): 

749 

750 if ampId2 == ampId: 

751 serialOSCorr[ampId, ampId2] = 1.0 

752 imageCorr[ampId, ampId2] = 1.0 

753 else: 

754 rawOverscan2 = overscan2.overscanImage.image.array + overscan2.overscanFit 

755 rawOverscan2 = rawOverscan2.ravel() 

756 

757 serialOSCorr[ampId, ampId2] = np.corrcoef(rawOverscan, rawOverscan2)[0, 1] 

758 

759 ampImage2 = inputExp[detector[ampId2].getBBox()] 

760 ampImage2 = ampImage2.image.array.ravel() 

761 

762 imageCorr[ampId, ampId2] = np.corrcoef(ampImage, ampImage2)[0, 1] 

763 

764 outputStats["OVERSCAN_CORR"] = serialOSCorr.tolist() 

765 outputStats["IMAGE_CORR"] = imageCorr.tolist() 

766 

767 return outputStats