Coverage for python/lsst/ip/isr/overscan.py: 14%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

200 statements  

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import lsst.afw.math as afwMath 

24import lsst.afw.image as afwImage 

25import lsst.pipe.base as pipeBase 

26import lsst.pex.config as pexConfig 

27 

28__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"] 

29 

30 

31class OverscanCorrectionTaskConfig(pexConfig.Config): 

32 """Overscan correction options. 

33 """ 

34 fitType = pexConfig.ChoiceField( 

35 dtype=str, 

36 doc="The method for fitting the overscan bias level.", 

37 default='MEDIAN', 

38 allowed={ 

39 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

40 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

41 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

42 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

43 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

44 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

45 "MEAN": "Correct using the mean of the overscan region", 

46 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

47 "MEDIAN": "Correct using the median of the overscan region", 

48 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

49 }, 

50 ) 

51 order = pexConfig.Field( 

52 dtype=int, 

53 doc=("Order of polynomial to fit if overscan fit type is a polynomial, " 

54 "or number of spline knots if overscan fit type is a spline."), 

55 default=1, 

56 ) 

57 numSigmaClip = pexConfig.Field( 

58 dtype=float, 

59 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

60 default=3.0, 

61 ) 

62 maskPlanes = pexConfig.ListField( 

63 dtype=str, 

64 doc="Mask planes to reject when measuring overscan", 

65 default=['SAT'], 

66 ) 

67 overscanIsInt = pexConfig.Field( 

68 dtype=bool, 

69 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN" 

70 " and fitType=MEDIAN_PER_ROW.", 

71 default=True, 

72 ) 

73 

74 

75class OverscanCorrectionTask(pipeBase.Task): 

76 """Correction task for overscan. 

77 

78 This class contains a number of utilities that are easier to 

79 understand and use when they are not embedded in nested if/else 

80 loops. 

81 

82 Parameters 

83 ---------- 

84 statControl : `lsst.afw.math.StatisticsControl`, optional 

85 Statistics control object. 

86 """ 

87 ConfigClass = OverscanCorrectionTaskConfig 

88 _DefaultName = "overscan" 

89 

90 def __init__(self, statControl=None, **kwargs): 

91 super().__init__(**kwargs) 

92 self.allowDebug = True 

93 

94 if statControl: 

95 self.statControl = statControl 

96 else: 

97 self.statControl = afwMath.StatisticsControl() 

98 self.statControl.setNumSigmaClip(self.config.numSigmaClip) 

99 self.statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes)) 

100 

101 def run(self, ampImage, overscanImage, amp=None): 

102 """Measure and remove an overscan from an amplifier image. 

103 

104 Parameters 

105 ---------- 

106 ampImage : `lsst.afw.image.Image` 

107 Image data that will have the overscan removed. 

108 overscanImage : `lsst.afw.image.Image` 

109 Overscan data that the overscan is measured from. 

110 amp : `lsst.afw.cameraGeom.Amplifier`, optional 

111 Amplifier to use for debugging purposes. 

112 

113 Returns 

114 ------- 

115 overscanResults : `lsst.pipe.base.Struct` 

116 Result struct with components: 

117 

118 ``imageFit`` 

119 Value or fit subtracted from the amplifier image data 

120 (scalar or `lsst.afw.image.Image`). 

121 ``overscanFit`` 

122 Value or fit subtracted from the overscan image data 

123 (scalar or `lsst.afw.image.Image`). 

124 ``overscanImage`` 

125 Image of the overscan region with the overscan 

126 correction applied (`lsst.afw.image.Image`). This 

127 quantity is used to estimate the amplifier read noise 

128 empirically. 

129 

130 Raises 

131 ------ 

132 RuntimeError 

133 Raised if an invalid overscan type is set. 

134 

135 """ 

136 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'): 

137 overscanResult = self.measureConstantOverscan(overscanImage) 

138 overscanValue = overscanResult.overscanValue 

139 offImage = overscanValue 

140 overscanModel = overscanValue 

141 maskSuspect = None 

142 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG', 

143 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'): 

144 overscanResult = self.measureVectorOverscan(overscanImage) 

145 overscanValue = overscanResult.overscanValue 

146 maskArray = overscanResult.maskArray 

147 isTransposed = overscanResult.isTransposed 

148 

149 offImage = afwImage.ImageF(ampImage.getDimensions()) 

150 offArray = offImage.getArray() 

151 overscanModel = afwImage.ImageF(overscanImage.getDimensions()) 

152 overscanArray = overscanModel.getArray() 

153 

154 if hasattr(ampImage, 'getMask'): 

155 maskSuspect = afwImage.Mask(ampImage.getDimensions()) 

156 else: 

157 maskSuspect = None 

158 

159 if isTransposed: 

160 offArray[:, :] = overscanValue[np.newaxis, :] 

161 overscanArray[:, :] = overscanValue[np.newaxis, :] 

162 if maskSuspect: 

163 maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT") 

164 else: 

165 offArray[:, :] = overscanValue[:, np.newaxis] 

166 overscanArray[:, :] = overscanValue[:, np.newaxis] 

167 if maskSuspect: 

168 maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT") 

169 else: 

170 raise RuntimeError('%s : %s an invalid overscan type' % 

171 ("overscanCorrection", self.config.fitType)) 

172 

173 self.debugView(overscanImage, overscanValue, amp) 

174 

175 ampImage -= offImage 

176 if maskSuspect: 

177 ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :] 

178 overscanImage -= overscanModel 

179 return pipeBase.Struct(imageFit=offImage, 

180 overscanFit=overscanModel, 

181 overscanImage=overscanImage, 

182 edgeMask=maskSuspect) 

183 

184 @staticmethod 

185 def integerConvert(image): 

186 """Return an integer version of the input image. 

187 

188 Parameters 

189 ---------- 

190 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

191 Image to convert to integers. 

192 

193 Returns 

194 ------- 

195 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

196 The integer converted image. 

197 

198 Raises 

199 ------ 

200 RuntimeError 

201 Raised if the input image could not be converted. 

202 """ 

203 if hasattr(image, "image"): 

204 # Is a maskedImage: 

205 imageI = image.image.convertI() 

206 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance) 

207 elif hasattr(image, "convertI"): 

208 # Is an Image: 

209 outI = image.convertI() 

210 elif hasattr(image, "astype"): 

211 # Is a numpy array: 

212 outI = image.astype(int) 

213 else: 

214 raise RuntimeError("Could not convert this to integers: %s %s %s", 

215 image, type(image), dir(image)) 

216 return outI 

217 

218 # Constant methods 

219 def measureConstantOverscan(self, image): 

220 """Measure a constant overscan value. 

221 

222 Parameters 

223 ---------- 

224 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

225 Image data to measure the overscan from. 

226 

227 Returns 

228 ------- 

229 results : `lsst.pipe.base.Struct` 

230 Overscan result with entries: 

231 - ``overscanValue``: Overscan value to subtract (`float`) 

232 - ``maskArray``: Placeholder for a mask array (`list`) 

233 - ``isTransposed``: Orientation of the overscan (`bool`) 

234 """ 

235 if self.config.fitType == 'MEDIAN': 

236 calcImage = self.integerConvert(image) 

237 else: 

238 calcImage = image 

239 

240 fitType = afwMath.stringToStatisticsProperty(self.config.fitType) 

241 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue() 

242 

243 return pipeBase.Struct(overscanValue=overscanValue, 

244 maskArray=None, 

245 isTransposed=False) 

246 

247 # Vector correction utilities 

248 def getImageArray(self, image): 

249 """Extract the numpy array from the input image. 

250 

251 Parameters 

252 ---------- 

253 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

254 Image data to pull array from. 

255 

256 calcImage : `numpy.ndarray` 

257 Image data array for numpy operating. 

258 """ 

259 if hasattr(image, "getImage"): 

260 calcImage = image.getImage().getArray() 

261 calcImage = np.ma.masked_where(image.getMask().getArray() & self.statControl.getAndMask(), 

262 calcImage) 

263 else: 

264 calcImage = image.getArray() 

265 return calcImage 

266 

267 @staticmethod 

268 def transpose(imageArray): 

269 """Transpose input numpy array if necessary. 

270 

271 Parameters 

272 ---------- 

273 imageArray : `numpy.ndarray` 

274 Image data to transpose. 

275 

276 Returns 

277 ------- 

278 imageArray : `numpy.ndarray` 

279 Transposed image data. 

280 isTransposed : `bool` 

281 Indicates whether the input data was transposed. 

282 """ 

283 if np.argmin(imageArray.shape) == 0: 

284 return np.transpose(imageArray), True 

285 else: 

286 return imageArray, False 

287 

288 def maskOutliers(self, imageArray): 

289 """Mask outliers in a row of overscan data from a robust sigma 

290 clipping procedure. 

291 

292 Parameters 

293 ---------- 

294 imageArray : `numpy.ndarray` 

295 Image to filter along numpy axis=1. 

296 

297 Returns 

298 ------- 

299 maskedArray : `numpy.ma.masked_array` 

300 Masked image marking outliers. 

301 """ 

302 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1) 

303 axisMedians = median 

304 axisStdev = 0.74*(uq - lq) # robust stdev 

305 

306 diff = np.abs(imageArray - axisMedians[:, np.newaxis]) 

307 return np.ma.masked_where(diff > self.statControl.getNumSigmaClip() 

308 * axisStdev[:, np.newaxis], imageArray) 

309 

310 @staticmethod 

311 def collapseArray(maskedArray): 

312 """Collapse overscan array (and mask) to a 1-D vector of values. 

313 

314 Parameters 

315 ---------- 

316 maskedArray : `numpy.ma.masked_array` 

317 Masked array of input overscan data. 

318 

319 Returns 

320 ------- 

321 collapsed : `numpy.ma.masked_array` 

322 Single dimensional overscan data, combined with the mean. 

323 """ 

324 collapsed = np.mean(maskedArray, axis=1) 

325 if collapsed.mask.sum() > 0: 

326 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1) 

327 return collapsed 

328 

329 def collapseArrayMedian(self, maskedArray): 

330 """Collapse overscan array (and mask) to a 1-D vector of using the 

331 correct integer median of row-values. 

332 

333 Parameters 

334 ---------- 

335 maskedArray : `numpy.ma.masked_array` 

336 Masked array of input overscan data. 

337 

338 Returns 

339 ------- 

340 collapsed : `numpy.ma.masked_array` 

341 Single dimensional overscan data, combined with the afwMath median. 

342 """ 

343 integerMI = self.integerConvert(maskedArray) 

344 

345 collapsed = [] 

346 fitType = afwMath.stringToStatisticsProperty('MEDIAN') 

347 for row in integerMI: 

348 newRow = row.compressed() 

349 if len(newRow) > 0: 

350 rowMedian = afwMath.makeStatistics(newRow, fitType, self.statControl).getValue() 

351 else: 

352 rowMedian = np.nan 

353 collapsed.append(rowMedian) 

354 

355 return np.array(collapsed) 

356 

357 def splineFit(self, indices, collapsed, numBins): 

358 """Wrapper function to match spline fit API to polynomial fit API. 

359 

360 Parameters 

361 ---------- 

362 indices : `numpy.ndarray` 

363 Locations to evaluate the spline. 

364 collapsed : `numpy.ndarray` 

365 Collapsed overscan values corresponding to the spline 

366 evaluation points. 

367 numBins : `int` 

368 Number of bins to use in constructing the spline. 

369 

370 Returns 

371 ------- 

372 interp : `lsst.afw.math.Interpolate` 

373 Interpolation object for later evaluation. 

374 """ 

375 if not np.ma.is_masked(collapsed): 

376 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask]) 

377 

378 numPerBin, binEdges = np.histogram(indices, bins=numBins, 

379 weights=1 - collapsed.mask.astype(int)) 

380 with np.errstate(invalid="ignore"): 

381 values = np.histogram(indices, bins=numBins, 

382 weights=collapsed.data*~collapsed.mask)[0]/numPerBin 

383 binCenters = np.histogram(indices, bins=numBins, 

384 weights=indices*~collapsed.mask)[0]/numPerBin 

385 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0], 

386 values.astype(float)[numPerBin > 0], 

387 afwMath.stringToInterpStyle(self.config.fitType)) 

388 return interp 

389 

390 @staticmethod 

391 def splineEval(indices, interp): 

392 """Wrapper function to match spline evaluation API to polynomial fit 

393 API. 

394 

395 Parameters 

396 ---------- 

397 indices : `numpy.ndarray` 

398 Locations to evaluate the spline. 

399 interp : `lsst.afw.math.interpolate` 

400 Interpolation object to use. 

401 

402 Returns 

403 ------- 

404 values : `numpy.ndarray` 

405 Evaluated spline values at each index. 

406 """ 

407 

408 return interp.interpolate(indices.astype(float)) 

409 

410 @staticmethod 

411 def maskExtrapolated(collapsed): 

412 """Create mask if edges are extrapolated. 

413 

414 Parameters 

415 ---------- 

416 collapsed : `numpy.ma.masked_array` 

417 Masked array to check the edges of. 

418 

419 Returns 

420 ------- 

421 maskArray : `numpy.ndarray` 

422 Boolean numpy array of pixels to mask. 

423 """ 

424 maskArray = np.full_like(collapsed, False, dtype=bool) 

425 if np.ma.is_masked(collapsed): 

426 num = len(collapsed) 

427 for low in range(num): 

428 if not collapsed.mask[low]: 

429 break 

430 if low > 0: 

431 maskArray[:low] = True 

432 for high in range(1, num): 

433 if not collapsed.mask[-high]: 

434 break 

435 if high > 1: 

436 maskArray[-high:] = True 

437 return maskArray 

438 

439 def measureVectorOverscan(self, image): 

440 """Calculate the 1-d vector overscan from the input overscan image. 

441 

442 Parameters 

443 ---------- 

444 image : `lsst.afw.image.MaskedImage` 

445 Image containing the overscan data. 

446 

447 Returns 

448 ------- 

449 results : `lsst.pipe.base.Struct` 

450 Overscan result with entries: 

451 - ``overscanValue``: Overscan value to subtract (`float`) 

452 - ``maskArray`` : `list` [ `bool` ] 

453 List of rows that should be masked as ``SUSPECT`` when the 

454 overscan solution is applied. 

455 - ``isTransposed`` : `bool` 

456 Indicates if the overscan data was transposed during 

457 calcuation, noting along which axis the overscan should be 

458 subtracted. 

459 """ 

460 calcImage = self.getImageArray(image) 

461 

462 # operate on numpy-arrays from here 

463 calcImage, isTransposed = self.transpose(calcImage) 

464 masked = self.maskOutliers(calcImage) 

465 

466 if self.config.fitType == 'MEDIAN_PER_ROW': 

467 overscanVector = self.collapseArrayMedian(masked) 

468 maskArray = self.maskExtrapolated(overscanVector) 

469 else: 

470 collapsed = self.collapseArray(masked) 

471 

472 num = len(collapsed) 

473 indices = 2.0*np.arange(num)/float(num) - 1.0 

474 

475 poly = np.polynomial 

476 fitter, evaler = { 

477 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval), 

478 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval), 

479 'LEG': (poly.legendre.legfit, poly.legendre.legval), 

480 'NATURAL_SPLINE': (self.splineFit, self.splineEval), 

481 'CUBIC_SPLINE': (self.splineFit, self.splineEval), 

482 'AKIMA_SPLINE': (self.splineFit, self.splineEval) 

483 }[self.config.fitType] 

484 

485 coeffs = fitter(indices, collapsed, self.config.order) 

486 overscanVector = evaler(indices, coeffs) 

487 maskArray = self.maskExtrapolated(collapsed) 

488 return pipeBase.Struct(overscanValue=np.array(overscanVector), 

489 maskArray=maskArray, 

490 isTransposed=isTransposed) 

491 

492 def debugView(self, image, model, amp=None): 

493 """Debug display for the final overscan solution. 

494 

495 Parameters 

496 ---------- 

497 image : `lsst.afw.image.Image` 

498 Input image the overscan solution was determined from. 

499 model : `numpy.ndarray` or `float` 

500 Overscan model determined for the image. 

501 amp : `lsst.afw.cameraGeom.Amplifier`, optional 

502 Amplifier to extract diagnostic information. 

503 """ 

504 import lsstDebug 

505 if not lsstDebug.Info(__name__).display: 

506 return 

507 if not self.allowDebug: 

508 return 

509 

510 calcImage = self.getImageArray(image) 

511 calcImage, isTransposed = self.transpose(calcImage) 

512 masked = self.maskOutliers(calcImage) 

513 collapsed = self.collapseArray(masked) 

514 

515 num = len(collapsed) 

516 indices = 2.0 * np.arange(num)/float(num) - 1.0 

517 

518 if np.ma.is_masked(collapsed): 

519 collapsedMask = collapsed.mask 

520 else: 

521 collapsedMask = np.array(num*[np.ma.nomask]) 

522 

523 import matplotlib.pyplot as plot 

524 figure = plot.figure(1) 

525 figure.clear() 

526 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

527 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+') 

528 if collapsedMask.sum() > 0: 

529 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+') 

530 if isinstance(model, np.ndarray): 

531 plotModel = model 

532 else: 

533 plotModel = np.zeros_like(indices) 

534 plotModel += model 

535 axes.plot(indices, plotModel, 'r-') 

536 plot.xlabel("centered/scaled position along overscan region") 

537 plot.ylabel("pixel value/fit value") 

538 if amp: 

539 plot.title(f"{amp.getName()} DataX: " 

540 f"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]" 

541 f"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:" 

542 f"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}") 

543 else: 

544 plot.title("No amp supplied.") 

545 figure.show() 

546 prompt = "Press Enter or c to continue [chp]..." 

547 while True: 

548 ans = input(prompt).lower() 

549 if ans in ("", " ", "c",): 

550 break 

551 elif ans in ("p", ): 

552 import pdb 

553 pdb.set_trace() 

554 elif ans in ('x', ): 

555 self.allowDebug = False 

556 break 

557 elif ans in ("h", ): 

558 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

559 plot.close()