Coverage for python/lsst/ip/isr/isrStatistics.py: 19%

153 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-08 11:41 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"] 

23 

24import numpy as np 

25 

26from scipy.signal.windows import hamming, hann, gaussian 

27 

28import lsst.afw.math as afwMath 

29import lsst.afw.image as afwImage 

30import lsst.pipe.base as pipeBase 

31import lsst.pex.config as pexConfig 

32 

33from lsst.afw.cameraGeom import ReadoutCorner 

34 

35 

36class IsrStatisticsTaskConfig(pexConfig.Config): 

37 """Image statistics options. 

38 """ 

39 doCtiStatistics = pexConfig.Field( 

40 dtype=bool, 

41 doc="Measure CTI statistics from image and overscans?", 

42 default=False, 

43 ) 

44 doApplyGainsForCtiStatistics = pexConfig.Field( 

45 dtype=bool, 

46 doc="Apply gain to the overscan region when measuring CTI statistics?", 

47 default=True, 

48 ) 

49 

50 doBandingStatistics = pexConfig.Field( 

51 dtype=bool, 

52 doc="Measure image banding metric?", 

53 default=False, 

54 ) 

55 bandingKernelSize = pexConfig.Field( 55 ↛ exitline 55 didn't jump to the function exit

56 dtype=int, 

57 doc="Width of box for boxcar smoothing for banding metric.", 

58 default=3, 

59 check=lambda x: x == 0 or x % 2 != 0, 

60 ) 

61 bandingFractionLow = pexConfig.Field( 61 ↛ exitline 61 didn't jump to the function exit

62 dtype=float, 

63 doc="Fraction of values to exclude from low samples.", 

64 default=0.1, 

65 check=lambda x: x >= 0.0 and x <= 1.0 

66 ) 

67 bandingFractionHigh = pexConfig.Field( 67 ↛ exitline 67 didn't jump to the function exit

68 dtype=float, 

69 doc="Fraction of values to exclude from high samples.", 

70 default=0.9, 

71 check=lambda x: x >= 0.0 and x <= 1.0, 

72 ) 

73 bandingUseHalfDetector = pexConfig.Field( 

74 dtype=float, 

75 doc="Use only the first half set of amplifiers.", 

76 default=True, 

77 ) 

78 

79 doProjectionStatistics = pexConfig.Field( 

80 dtype=bool, 

81 doc="Measure projection metric?", 

82 default=False, 

83 ) 

84 projectionKernelSize = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit

85 dtype=int, 

86 doc="Width of box for boxcar smoothing of projections.", 

87 default=0, 

88 check=lambda x: x == 0 or x % 2 != 0, 

89 ) 

90 doProjectionFft = pexConfig.Field( 

91 dtype=bool, 

92 doc="Generate FFTs from the image projections?", 

93 default=False, 

94 ) 

95 projectionFftWindow = pexConfig.ChoiceField( 

96 dtype=str, 

97 doc="Type of windowing to use prior to calculating FFT.", 

98 default="HAMMING", 

99 allowed={ 

100 "HAMMING": "Hamming window.", 

101 "HANN": "Hann window.", 

102 "GAUSSIAN": "Gaussian window.", 

103 "NONE": "No window." 

104 } 

105 ) 

106 

107 stat = pexConfig.Field( 

108 dtype=str, 

109 default='MEANCLIP', 

110 doc="Statistic name to use to measure regions.", 

111 ) 

112 nSigmaClip = pexConfig.Field( 

113 dtype=float, 

114 default=3.0, 

115 doc="Clipping threshold for background", 

116 ) 

117 nIter = pexConfig.Field( 

118 dtype=int, 

119 default=3, 

120 doc="Clipping iterations for background", 

121 ) 

122 badMask = pexConfig.ListField( 

123 dtype=str, 

124 default=["BAD", "INTRP", "SAT"], 

125 doc="Mask planes to ignore when identifying source pixels." 

126 ) 

127 

128 

129class IsrStatisticsTask(pipeBase.Task): 

130 """Task to measure arbitrary statistics on ISR processed exposures. 

131 

132 The goal is to wrap a number of optional measurements that are 

133 useful for calibration production and detector stability. 

134 """ 

135 ConfigClass = IsrStatisticsTaskConfig 

136 _DefaultName = "isrStatistics" 

137 

138 def __init__(self, statControl=None, **kwargs): 

139 super().__init__(**kwargs) 

140 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter, 

141 afwImage.Mask.getPlaneBitMask(self.config.badMask)) 

142 self.statType = afwMath.stringToStatisticsProperty(self.config.stat) 

143 

144 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs): 

145 """Task to run arbitrary statistics. 

146 

147 The statistics should be measured by individual methods, and 

148 add to the dictionary in the return struct. 

149 

150 Parameters 

151 ---------- 

152 inputExp : `lsst.afw.image.Exposure` 

153 The exposure to measure. 

154 ptc : `lsst.ip.isr.PtcDataset`, optional 

155 A PTC object containing gains to use. 

156 overscanResults : `list` [`lsst.pipe.base.Struct`], optional 

157 List of overscan results. Expected fields are: 

158 

159 ``imageFit`` 

160 Value or fit subtracted from the amplifier image data 

161 (scalar or `lsst.afw.image.Image`). 

162 ``overscanFit`` 

163 Value or fit subtracted from the overscan image data 

164 (scalar or `lsst.afw.image.Image`). 

165 ``overscanImage`` 

166 Image of the overscan region with the overscan 

167 correction applied (`lsst.afw.image.Image`). This 

168 quantity is used to estimate the amplifier read noise 

169 empirically. 

170 

171 Returns 

172 ------- 

173 resultStruct : `lsst.pipe.base.Struct` 

174 Contains the measured statistics as a dict stored in a 

175 field named ``results``. 

176 

177 Raises 

178 ------ 

179 RuntimeError 

180 Raised if the amplifier gains could not be found. 

181 """ 

182 # Find gains. 

183 detector = inputExp.getDetector() 

184 if ptc is not None: 

185 gains = ptc.gain 

186 elif detector is not None: 

187 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()} 

188 else: 

189 raise RuntimeError("No source of gains provided.") 

190 

191 ctiResults = None 

192 if self.config.doCtiStatistics: 

193 ctiResults = self.measureCti(inputExp, overscanResults, gains) 

194 

195 bandingResults = None 

196 if self.config.doBandingStatistics: 

197 bandingResults = self.measureBanding(inputExp, overscanResults) 

198 

199 projectionResults = None 

200 if self.config.doProjectionStatistics: 

201 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults) 

202 

203 return pipeBase.Struct( 

204 results={'CTI': ctiResults, 

205 'BANDING': bandingResults, 

206 'PROJECTION': projectionResults, 

207 }, 

208 ) 

209 

210 def measureCti(self, inputExp, overscans, gains): 

211 """Task to measure CTI statistics. 

212 

213 Parameters 

214 ---------- 

215 inputExp : `lsst.afw.image.Exposure` 

216 Exposure to measure. 

217 overscans : `list` [`lsst.pipe.base.Struct`] 

218 List of overscan results. Expected fields are: 

219 

220 ``imageFit`` 

221 Value or fit subtracted from the amplifier image data 

222 (scalar or `lsst.afw.image.Image`). 

223 ``overscanFit`` 

224 Value or fit subtracted from the overscan image data 

225 (scalar or `lsst.afw.image.Image`). 

226 ``overscanImage`` 

227 Image of the overscan region with the overscan 

228 correction applied (`lsst.afw.image.Image`). This 

229 quantity is used to estimate the amplifier read noise 

230 empirically. 

231 gains : `dict` [`str` `float`] 

232 Dictionary of per-amplifier gains, indexed by amplifier name. 

233 

234 Returns 

235 ------- 

236 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

237 Dictionary of measurements, keyed by amplifier name and 

238 statistics segment. 

239 """ 

240 outputStats = {} 

241 

242 detector = inputExp.getDetector() 

243 image = inputExp.image 

244 

245 # Ensure we have the same number of overscans as amplifiers. 

246 assert len(overscans) == len(detector.getAmplifiers()) 

247 

248 for ampIter, amp in enumerate(detector.getAmplifiers()): 

249 ampStats = {} 

250 gain = gains[amp.getName()] 

251 readoutCorner = amp.getReadoutCorner() 

252 # Full data region. 

253 dataRegion = image[amp.getBBox()] 

254 ampStats['IMAGE_MEAN'] = afwMath.makeStatistics(dataRegion, self.statType, 

255 self.statControl).getValue() 

256 

257 # First and last image columns. 

258 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0], 

259 self.statType, 

260 self.statControl).getValue() 

261 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1], 

262 self.statType, 

263 self.statControl).getValue() 

264 

265 # We want these relative to the readout corner. If that's 

266 # on the right side, we need to swap them. 

267 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

268 ampStats['FIRST_MEAN'] = pixelZ 

269 ampStats['LAST_MEAN'] = pixelA 

270 else: 

271 ampStats['FIRST_MEAN'] = pixelA 

272 ampStats['LAST_MEAN'] = pixelZ 

273 

274 # Measure the columns of the overscan. 

275 if overscans[ampIter] is None: 

276 # The amplifier is likely entirely bad, and needs to 

277 # be skipped. 

278 self.log.warn("No overscan information available for ISR statistics for amp %s.", 

279 amp.getName()) 

280 nCols = amp.getSerialOverscanBBox().getWidth() 

281 ampStats['OVERSCAN_COLUMNS'] = np.full((nCols, ), np.nan) 

282 ampStats['OVERSCAN_VALUES'] = np.full((nCols, ), np.nan) 

283 else: 

284 overscanImage = overscans[ampIter].overscanImage 

285 columns = [] 

286 values = [] 

287 for column in range(0, overscanImage.getWidth()): 

288 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column], 

289 self.statType, self.statControl).getValue() 

290 columns.append(column) 

291 if self.config.doApplyGainsForCtiStatistics: 

292 values.append(gain * osMean) 

293 else: 

294 values.append(osMean) 

295 

296 # We want these relative to the readout corner. If that's 

297 # on the right side, we need to swap them. 

298 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

299 ampStats['OVERSCAN_COLUMNS'] = list(reversed(columns)) 

300 ampStats['OVERSCAN_VALUES'] = list(reversed(values)) 

301 else: 

302 ampStats['OVERSCAN_COLUMNS'] = columns 

303 ampStats['OVERSCAN_VALUES'] = values 

304 

305 outputStats[amp.getName()] = ampStats 

306 

307 return outputStats 

308 

309 @staticmethod 

310 def makeKernel(kernelSize): 

311 """Make a boxcar smoothing kernel. 

312 

313 Parameters 

314 ---------- 

315 kernelSize : `int` 

316 Size of the kernel in pixels. 

317 

318 Returns 

319 ------- 

320 kernel : `np.array` 

321 Kernel for boxcar smoothing. 

322 """ 

323 if kernelSize > 0: 

324 kernel = np.full(kernelSize, 1.0 / kernelSize) 

325 else: 

326 kernel = np.array([1.0]) 

327 return kernel 

328 

329 def measureBanding(self, inputExp, overscans): 

330 """Task to measure banding statistics. 

331 

332 Parameters 

333 ---------- 

334 inputExp : `lsst.afw.image.Exposure` 

335 Exposure to measure. 

336 overscans : `list` [`lsst.pipe.base.Struct`] 

337 List of overscan results. Expected fields are: 

338 

339 ``imageFit`` 

340 Value or fit subtracted from the amplifier image data 

341 (scalar or `lsst.afw.image.Image`). 

342 ``overscanFit`` 

343 Value or fit subtracted from the overscan image data 

344 (scalar or `lsst.afw.image.Image`). 

345 ``overscanImage`` 

346 Image of the overscan region with the overscan 

347 correction applied (`lsst.afw.image.Image`). This 

348 quantity is used to estimate the amplifier read noise 

349 empirically. 

350 

351 Returns 

352 ------- 

353 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

354 Dictionary of measurements, keyed by amplifier name and 

355 statistics segment. 

356 """ 

357 outputStats = {} 

358 

359 detector = inputExp.getDetector() 

360 kernel = self.makeKernel(self.config.bandingKernelSize) 

361 

362 outputStats['AMP_BANDING'] = [] 

363 for amp, overscanData in zip(detector.getAmplifiers(), overscans): 

364 overscanFit = np.array(overscanData.overscanFit) 

365 overscanArray = overscanData.overscanImage.image.array 

366 rawOverscan = np.mean(overscanArray + overscanFit, axis=1) 

367 

368 smoothedOverscan = np.convolve(rawOverscan, kernel, mode='valid') 

369 

370 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow, 

371 self.config.bandingFractionHigh]) 

372 outputStats['AMP_BANDING'].append(float(high - low)) 

373 

374 if self.config.bandingUseHalfDetector: 

375 fullLength = len(outputStats['AMP_BANDING']) 

376 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING'][0:fullLength//2])) 

377 else: 

378 outputStats['DET_BANDING'] = float(np.nanmedian(outputStats['AMP_BANDING'])) 

379 

380 return outputStats 

381 

382 def measureProjectionStatistics(self, inputExp, overscans): 

383 """Task to measure metrics from image slicing. 

384 

385 Parameters 

386 ---------- 

387 inputExp : `lsst.afw.image.Exposure` 

388 Exposure to measure. 

389 overscans : `list` [`lsst.pipe.base.Struct`] 

390 List of overscan results. Expected fields are: 

391 

392 ``imageFit`` 

393 Value or fit subtracted from the amplifier image data 

394 (scalar or `lsst.afw.image.Image`). 

395 ``overscanFit`` 

396 Value or fit subtracted from the overscan image data 

397 (scalar or `lsst.afw.image.Image`). 

398 ``overscanImage`` 

399 Image of the overscan region with the overscan 

400 correction applied (`lsst.afw.image.Image`). This 

401 quantity is used to estimate the amplifier read noise 

402 empirically. 

403 

404 Returns 

405 ------- 

406 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

407 Dictionary of measurements, keyed by amplifier name and 

408 statistics segment. 

409 """ 

410 outputStats = {} 

411 

412 detector = inputExp.getDetector() 

413 kernel = self.makeKernel(self.config.projectionKernelSize) 

414 

415 outputStats['AMP_VPROJECTION'] = {} 

416 outputStats['AMP_HPROJECTION'] = {} 

417 convolveMode = 'valid' 

418 if self.config.doProjectionFft: 

419 outputStats['AMP_VFFT_REAL'] = {} 

420 outputStats['AMP_VFFT_IMAG'] = {} 

421 outputStats['AMP_HFFT_REAL'] = {} 

422 outputStats['AMP_HFFT_IMAG'] = {} 

423 convolveMode = 'same' 

424 

425 for amp in detector.getAmplifiers(): 

426 ampArray = inputExp.image[amp.getBBox()].array 

427 

428 horizontalProjection = np.mean(ampArray, axis=0) 

429 verticalProjection = np.mean(ampArray, axis=1) 

430 

431 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode) 

432 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode) 

433 

434 outputStats['AMP_HPROJECTION'][amp.getName()] = horizontalProjection.tolist() 

435 outputStats['AMP_VPROJECTION'][amp.getName()] = verticalProjection.tolist() 

436 

437 if self.config.doProjectionFft: 

438 horizontalWindow = np.ones_like(horizontalProjection) 

439 verticalWindow = np.ones_like(verticalProjection) 

440 if self.config.projectionFftWindow == "NONE": 

441 pass 

442 elif self.config.projectionFftWindow == "HAMMING": 

443 horizontalWindow = hamming(len(horizontalProjection)) 

444 verticalWindow = hamming(len(verticalProjection)) 

445 elif self.config.projectionFftWindow == "HANN": 

446 horizontalWindow = hann(len(horizontalProjection)) 

447 verticalWindow = hann(len(verticalProjection)) 

448 elif self.config.projectionFftWindow == "GAUSSIAN": 

449 horizontalWindow = gaussian(len(horizontalProjection)) 

450 verticalWindow = gaussian(len(verticalProjection)) 

451 else: 

452 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}") 

453 

454 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow)) 

455 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow)) 

456 outputStats['AMP_HFFT_REAL'][amp.getName()] = np.real(horizontalFFT).tolist() 

457 outputStats['AMP_HFFT_IMAG'][amp.getName()] = np.imag(horizontalFFT).tolist() 

458 outputStats['AMP_VFFT_REAL'][amp.getName()] = np.real(verticalFFT).tolist() 

459 outputStats['AMP_VFFT_IMAG'][amp.getName()] = np.imag(verticalFFT).tolist() 

460 

461 return outputStats