Coverage for python/lsst/ip/isr/isrStatistics.py: 18%

170 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-23 11:36 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IsrStatisticsTaskConfig", "IsrStatisticsTask"] 

23 

24import numpy as np 

25 

26from scipy.signal.windows import hamming, hann, gaussian 

27 

28import lsst.afw.math as afwMath 

29import lsst.afw.image as afwImage 

30import lsst.pipe.base as pipeBase 

31import lsst.pex.config as pexConfig 

32 

33from lsst.afw.cameraGeom import ReadoutCorner 

34 

35 

36class IsrStatisticsTaskConfig(pexConfig.Config): 

37 """Image statistics options. 

38 """ 

39 doCtiStatistics = pexConfig.Field( 

40 dtype=bool, 

41 doc="Measure CTI statistics from image and overscans?", 

42 default=False, 

43 ) 

44 doApplyGainsForCtiStatistics = pexConfig.Field( 

45 dtype=bool, 

46 doc="Apply gain to the overscan region when measuring CTI statistics?", 

47 default=True, 

48 ) 

49 

50 doBandingStatistics = pexConfig.Field( 

51 dtype=bool, 

52 doc="Measure image banding metric?", 

53 default=False, 

54 ) 

55 bandingKernelSize = pexConfig.Field( 55 ↛ exitline 55 didn't jump to the function exit

56 dtype=int, 

57 doc="Width of box for boxcar smoothing for banding metric.", 

58 default=3, 

59 check=lambda x: x == 0 or x % 2 != 0, 

60 ) 

61 bandingFractionLow = pexConfig.Field( 61 ↛ exitline 61 didn't jump to the function exit

62 dtype=float, 

63 doc="Fraction of values to exclude from low samples.", 

64 default=0.1, 

65 check=lambda x: x >= 0.0 and x <= 1.0 

66 ) 

67 bandingFractionHigh = pexConfig.Field( 67 ↛ exitline 67 didn't jump to the function exit

68 dtype=float, 

69 doc="Fraction of values to exclude from high samples.", 

70 default=0.9, 

71 check=lambda x: x >= 0.0 and x <= 1.0, 

72 ) 

73 bandingUseHalfDetector = pexConfig.Field( 

74 dtype=float, 

75 doc="Use only the first half set of amplifiers.", 

76 default=True, 

77 ) 

78 

79 doProjectionStatistics = pexConfig.Field( 

80 dtype=bool, 

81 doc="Measure projection metric?", 

82 default=False, 

83 ) 

84 projectionKernelSize = pexConfig.Field( 84 ↛ exitline 84 didn't jump to the function exit

85 dtype=int, 

86 doc="Width of box for boxcar smoothing of projections.", 

87 default=0, 

88 check=lambda x: x == 0 or x % 2 != 0, 

89 ) 

90 doProjectionFft = pexConfig.Field( 

91 dtype=bool, 

92 doc="Generate FFTs from the image projections?", 

93 default=False, 

94 ) 

95 projectionFftWindow = pexConfig.ChoiceField( 

96 dtype=str, 

97 doc="Type of windowing to use prior to calculating FFT.", 

98 default="HAMMING", 

99 allowed={ 

100 "HAMMING": "Hamming window.", 

101 "HANN": "Hann window.", 

102 "GAUSSIAN": "Gaussian window.", 

103 "NONE": "No window." 

104 } 

105 ) 

106 

107 doCopyCalibDistributionStatistics = pexConfig.Field( 

108 dtype=bool, 

109 doc="Copy calibration distribution statistics to output?", 

110 default=False, 

111 ) 

112 expectedDistributionLevels = pexConfig.ListField( 

113 dtype=float, 

114 doc="Percentile levels expected in the calibration header.", 

115 default=[0, 5, 16, 50, 84, 95, 100], 

116 ) 

117 

118 stat = pexConfig.Field( 

119 dtype=str, 

120 default="MEANCLIP", 

121 doc="Statistic name to use to measure regions.", 

122 ) 

123 nSigmaClip = pexConfig.Field( 

124 dtype=float, 

125 default=3.0, 

126 doc="Clipping threshold for background", 

127 ) 

128 nIter = pexConfig.Field( 

129 dtype=int, 

130 default=3, 

131 doc="Clipping iterations for background", 

132 ) 

133 badMask = pexConfig.ListField( 

134 dtype=str, 

135 default=["BAD", "INTRP", "SAT"], 

136 doc="Mask planes to ignore when identifying source pixels." 

137 ) 

138 

139 

140class IsrStatisticsTask(pipeBase.Task): 

141 """Task to measure arbitrary statistics on ISR processed exposures. 

142 

143 The goal is to wrap a number of optional measurements that are 

144 useful for calibration production and detector stability. 

145 """ 

146 ConfigClass = IsrStatisticsTaskConfig 

147 _DefaultName = "isrStatistics" 

148 

149 def __init__(self, statControl=None, **kwargs): 

150 super().__init__(**kwargs) 

151 self.statControl = afwMath.StatisticsControl(self.config.nSigmaClip, self.config.nIter, 

152 afwImage.Mask.getPlaneBitMask(self.config.badMask)) 

153 self.statType = afwMath.stringToStatisticsProperty(self.config.stat) 

154 

155 def run(self, inputExp, ptc=None, overscanResults=None, **kwargs): 

156 """Task to run arbitrary statistics. 

157 

158 The statistics should be measured by individual methods, and 

159 add to the dictionary in the return struct. 

160 

161 Parameters 

162 ---------- 

163 inputExp : `lsst.afw.image.Exposure` 

164 The exposure to measure. 

165 ptc : `lsst.ip.isr.PtcDataset`, optional 

166 A PTC object containing gains to use. 

167 overscanResults : `list` [`lsst.pipe.base.Struct`], optional 

168 List of overscan results. Expected fields are: 

169 

170 ``imageFit`` 

171 Value or fit subtracted from the amplifier image data 

172 (scalar or `lsst.afw.image.Image`). 

173 ``overscanFit`` 

174 Value or fit subtracted from the overscan image data 

175 (scalar or `lsst.afw.image.Image`). 

176 ``overscanImage`` 

177 Image of the overscan region with the overscan 

178 correction applied (`lsst.afw.image.Image`). This 

179 quantity is used to estimate the amplifier read noise 

180 empirically. 

181 

182 Returns 

183 ------- 

184 resultStruct : `lsst.pipe.base.Struct` 

185 Contains the measured statistics as a dict stored in a 

186 field named ``results``. 

187 

188 Raises 

189 ------ 

190 RuntimeError 

191 Raised if the amplifier gains could not be found. 

192 """ 

193 # Find gains. 

194 detector = inputExp.getDetector() 

195 if ptc is not None: 

196 gains = ptc.gain 

197 elif detector is not None: 

198 gains = {amp.getName(): amp.getGain() for amp in detector.getAmplifiers()} 

199 else: 

200 raise RuntimeError("No source of gains provided.") 

201 

202 ctiResults = None 

203 if self.config.doCtiStatistics: 

204 ctiResults = self.measureCti(inputExp, overscanResults, gains) 

205 

206 bandingResults = None 

207 if self.config.doBandingStatistics: 

208 bandingResults = self.measureBanding(inputExp, overscanResults) 

209 

210 projectionResults = None 

211 if self.config.doProjectionStatistics: 

212 projectionResults = self.measureProjectionStatistics(inputExp, overscanResults) 

213 

214 calibDistributionResults = None 

215 if self.config.doCopyCalibDistributionStatistics: 

216 calibDistributionResults = self.copyCalibDistributionStatistics(inputExp, **kwargs) 

217 

218 return pipeBase.Struct( 

219 results={"CTI": ctiResults, 

220 "BANDING": bandingResults, 

221 "PROJECTION": projectionResults, 

222 "CALIBDIST": calibDistributionResults, 

223 }, 

224 ) 

225 

226 def measureCti(self, inputExp, overscans, gains): 

227 """Task to measure CTI statistics. 

228 

229 Parameters 

230 ---------- 

231 inputExp : `lsst.afw.image.Exposure` 

232 Exposure to measure. 

233 overscans : `list` [`lsst.pipe.base.Struct`] 

234 List of overscan results. Expected fields are: 

235 

236 ``imageFit`` 

237 Value or fit subtracted from the amplifier image data 

238 (scalar or `lsst.afw.image.Image`). 

239 ``overscanFit`` 

240 Value or fit subtracted from the overscan image data 

241 (scalar or `lsst.afw.image.Image`). 

242 ``overscanImage`` 

243 Image of the overscan region with the overscan 

244 correction applied (`lsst.afw.image.Image`). This 

245 quantity is used to estimate the amplifier read noise 

246 empirically. 

247 gains : `dict` [`str` `float`] 

248 Dictionary of per-amplifier gains, indexed by amplifier name. 

249 

250 Returns 

251 ------- 

252 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

253 Dictionary of measurements, keyed by amplifier name and 

254 statistics segment. 

255 """ 

256 outputStats = {} 

257 

258 detector = inputExp.getDetector() 

259 image = inputExp.image 

260 

261 # Ensure we have the same number of overscans as amplifiers. 

262 assert len(overscans) == len(detector.getAmplifiers()) 

263 

264 for ampIter, amp in enumerate(detector.getAmplifiers()): 

265 ampStats = {} 

266 gain = gains[amp.getName()] 

267 readoutCorner = amp.getReadoutCorner() 

268 # Full data region. 

269 dataRegion = image[amp.getBBox()] 

270 ampStats["IMAGE_MEAN"] = afwMath.makeStatistics(dataRegion, self.statType, 

271 self.statControl).getValue() 

272 

273 # First and last image columns. 

274 pixelA = afwMath.makeStatistics(dataRegion.array[:, 0], 

275 self.statType, 

276 self.statControl).getValue() 

277 pixelZ = afwMath.makeStatistics(dataRegion.array[:, -1], 

278 self.statType, 

279 self.statControl).getValue() 

280 

281 # We want these relative to the readout corner. If that's 

282 # on the right side, we need to swap them. 

283 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

284 ampStats["FIRST_MEAN"] = pixelZ 

285 ampStats["LAST_MEAN"] = pixelA 

286 else: 

287 ampStats["FIRST_MEAN"] = pixelA 

288 ampStats["LAST_MEAN"] = pixelZ 

289 

290 # Measure the columns of the overscan. 

291 if overscans[ampIter] is None: 

292 # The amplifier is likely entirely bad, and needs to 

293 # be skipped. 

294 self.log.warning("No overscan information available for ISR statistics for amp %s.", 

295 amp.getName()) 

296 nCols = amp.getSerialOverscanBBox().getWidth() 

297 ampStats["OVERSCAN_COLUMNS"] = np.full((nCols, ), np.nan) 

298 ampStats["OVERSCAN_VALUES"] = np.full((nCols, ), np.nan) 

299 else: 

300 overscanImage = overscans[ampIter].overscanImage 

301 columns = [] 

302 values = [] 

303 for column in range(0, overscanImage.getWidth()): 

304 osMean = afwMath.makeStatistics(overscanImage.image.array[:, column], 

305 self.statType, self.statControl).getValue() 

306 columns.append(column) 

307 if self.config.doApplyGainsForCtiStatistics: 

308 values.append(gain * osMean) 

309 else: 

310 values.append(osMean) 

311 

312 # We want these relative to the readout corner. If that's 

313 # on the right side, we need to swap them. 

314 if readoutCorner in (ReadoutCorner.LR, ReadoutCorner.UR): 

315 ampStats["OVERSCAN_COLUMNS"] = list(reversed(columns)) 

316 ampStats["OVERSCAN_VALUES"] = list(reversed(values)) 

317 else: 

318 ampStats["OVERSCAN_COLUMNS"] = columns 

319 ampStats["OVERSCAN_VALUES"] = values 

320 

321 outputStats[amp.getName()] = ampStats 

322 

323 return outputStats 

324 

325 @staticmethod 

326 def makeKernel(kernelSize): 

327 """Make a boxcar smoothing kernel. 

328 

329 Parameters 

330 ---------- 

331 kernelSize : `int` 

332 Size of the kernel in pixels. 

333 

334 Returns 

335 ------- 

336 kernel : `np.array` 

337 Kernel for boxcar smoothing. 

338 """ 

339 if kernelSize > 0: 

340 kernel = np.full(kernelSize, 1.0 / kernelSize) 

341 else: 

342 kernel = np.array([1.0]) 

343 return kernel 

344 

345 def measureBanding(self, inputExp, overscans): 

346 """Task to measure banding statistics. 

347 

348 Parameters 

349 ---------- 

350 inputExp : `lsst.afw.image.Exposure` 

351 Exposure to measure. 

352 overscans : `list` [`lsst.pipe.base.Struct`] 

353 List of overscan results. Expected fields are: 

354 

355 ``imageFit`` 

356 Value or fit subtracted from the amplifier image data 

357 (scalar or `lsst.afw.image.Image`). 

358 ``overscanFit`` 

359 Value or fit subtracted from the overscan image data 

360 (scalar or `lsst.afw.image.Image`). 

361 ``overscanImage`` 

362 Image of the overscan region with the overscan 

363 correction applied (`lsst.afw.image.Image`). This 

364 quantity is used to estimate the amplifier read noise 

365 empirically. 

366 

367 Returns 

368 ------- 

369 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

370 Dictionary of measurements, keyed by amplifier name and 

371 statistics segment. 

372 """ 

373 outputStats = {} 

374 

375 detector = inputExp.getDetector() 

376 kernel = self.makeKernel(self.config.bandingKernelSize) 

377 

378 outputStats["AMP_BANDING"] = [] 

379 for amp, overscanData in zip(detector.getAmplifiers(), overscans): 

380 overscanFit = np.array(overscanData.overscanFit) 

381 overscanArray = overscanData.overscanImage.image.array 

382 rawOverscan = np.mean(overscanArray + overscanFit, axis=1) 

383 

384 smoothedOverscan = np.convolve(rawOverscan, kernel, mode="valid") 

385 

386 low, high = np.quantile(smoothedOverscan, [self.config.bandingFractionLow, 

387 self.config.bandingFractionHigh]) 

388 outputStats["AMP_BANDING"].append(float(high - low)) 

389 

390 if self.config.bandingUseHalfDetector: 

391 fullLength = len(outputStats["AMP_BANDING"]) 

392 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"][0:fullLength//2])) 

393 else: 

394 outputStats["DET_BANDING"] = float(np.nanmedian(outputStats["AMP_BANDING"])) 

395 

396 return outputStats 

397 

398 def measureProjectionStatistics(self, inputExp, overscans): 

399 """Task to measure metrics from image slicing. 

400 

401 Parameters 

402 ---------- 

403 inputExp : `lsst.afw.image.Exposure` 

404 Exposure to measure. 

405 overscans : `list` [`lsst.pipe.base.Struct`] 

406 List of overscan results. Expected fields are: 

407 

408 ``imageFit`` 

409 Value or fit subtracted from the amplifier image data 

410 (scalar or `lsst.afw.image.Image`). 

411 ``overscanFit`` 

412 Value or fit subtracted from the overscan image data 

413 (scalar or `lsst.afw.image.Image`). 

414 ``overscanImage`` 

415 Image of the overscan region with the overscan 

416 correction applied (`lsst.afw.image.Image`). This 

417 quantity is used to estimate the amplifier read noise 

418 empirically. 

419 

420 Returns 

421 ------- 

422 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

423 Dictionary of measurements, keyed by amplifier name and 

424 statistics segment. 

425 """ 

426 outputStats = {} 

427 

428 detector = inputExp.getDetector() 

429 kernel = self.makeKernel(self.config.projectionKernelSize) 

430 

431 outputStats["AMP_VPROJECTION"] = {} 

432 outputStats["AMP_HPROJECTION"] = {} 

433 convolveMode = "valid" 

434 if self.config.doProjectionFft: 

435 outputStats["AMP_VFFT_REAL"] = {} 

436 outputStats["AMP_VFFT_IMAG"] = {} 

437 outputStats["AMP_HFFT_REAL"] = {} 

438 outputStats["AMP_HFFT_IMAG"] = {} 

439 convolveMode = "same" 

440 

441 for amp in detector.getAmplifiers(): 

442 ampArray = inputExp.image[amp.getBBox()].array 

443 

444 horizontalProjection = np.mean(ampArray, axis=0) 

445 verticalProjection = np.mean(ampArray, axis=1) 

446 

447 horizontalProjection = np.convolve(horizontalProjection, kernel, mode=convolveMode) 

448 verticalProjection = np.convolve(verticalProjection, kernel, mode=convolveMode) 

449 

450 outputStats["AMP_HPROJECTION"][amp.getName()] = horizontalProjection.tolist() 

451 outputStats["AMP_VPROJECTION"][amp.getName()] = verticalProjection.tolist() 

452 

453 if self.config.doProjectionFft: 

454 horizontalWindow = np.ones_like(horizontalProjection) 

455 verticalWindow = np.ones_like(verticalProjection) 

456 if self.config.projectionFftWindow == "NONE": 

457 pass 

458 elif self.config.projectionFftWindow == "HAMMING": 

459 horizontalWindow = hamming(len(horizontalProjection)) 

460 verticalWindow = hamming(len(verticalProjection)) 

461 elif self.config.projectionFftWindow == "HANN": 

462 horizontalWindow = hann(len(horizontalProjection)) 

463 verticalWindow = hann(len(verticalProjection)) 

464 elif self.config.projectionFftWindow == "GAUSSIAN": 

465 horizontalWindow = gaussian(len(horizontalProjection)) 

466 verticalWindow = gaussian(len(verticalProjection)) 

467 else: 

468 raise RuntimeError(f"Invalid window function: {self.config.projectionFftWindow}") 

469 

470 horizontalFFT = np.fft.rfft(np.multiply(horizontalProjection, horizontalWindow)) 

471 verticalFFT = np.fft.rfft(np.multiply(verticalProjection, verticalWindow)) 

472 outputStats["AMP_HFFT_REAL"][amp.getName()] = np.real(horizontalFFT).tolist() 

473 outputStats["AMP_HFFT_IMAG"][amp.getName()] = np.imag(horizontalFFT).tolist() 

474 outputStats["AMP_VFFT_REAL"][amp.getName()] = np.real(verticalFFT).tolist() 

475 outputStats["AMP_VFFT_IMAG"][amp.getName()] = np.imag(verticalFFT).tolist() 

476 

477 return outputStats 

478 

479 def copyCalibDistributionStatistics(self, inputExp, **kwargs): 

480 """Copy calibration statistics for this exposure. 

481 

482 Parameters 

483 ---------- 

484 inputExp : `lsst.afw.image.Exposure` 

485 The exposure being processed. 

486 **kwargs : 

487 Keyword arguments with calibrations. 

488 

489 Returns 

490 ------- 

491 outputStats : `dict` [`str`, [`dict` [`str`,`float]] 

492 Dictionary of measurements, keyed by amplifier name and 

493 statistics segment. 

494 """ 

495 outputStats = {} 

496 

497 for amp in inputExp.getDetector(): 

498 ampStats = {} 

499 

500 for calibType in ("bias", "dark", "flat"): 

501 if kwargs.get(calibType, None) is not None: 

502 metadata = kwargs[calibType].getMetadata() 

503 for pct in self.config.expectedDistributionLevels: 

504 key = f"LSST CALIB {calibType.upper()} {amp.getName()} DISTRIBUTION {pct}-PCT" 

505 ampStats[key] = metadata.get(key, np.nan) 

506 outputStats[amp.getName()] = ampStats 

507 

508 return outputStats