Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import lsst.afw.math as afwMath 

24import lsst.afw.image as afwImage 

25import lsst.pipe.base as pipeBase 

26import lsst.pex.config as pexConfig 

27 

28__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"] 

29 

30 

31class OverscanCorrectionTaskConfig(pexConfig.Config): 

32 """Overscan correction options. 

33 """ 

34 fitType = pexConfig.ChoiceField( 

35 dtype=str, 

36 doc="The method for fitting the overscan bias level.", 

37 default='MEDIAN', 

38 allowed={ 

39 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

40 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

41 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

42 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

43 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

44 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

45 "MEAN": "Correct using the mean of the overscan region", 

46 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

47 "MEDIAN": "Correct using the median of the overscan region", 

48 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

49 }, 

50 ) 

51 order = pexConfig.Field( 

52 dtype=int, 

53 doc=("Order of polynomial to fit if overscan fit type is a polynomial, " + 

54 "or number of spline knots if overscan fit type is a spline."), 

55 default=1, 

56 ) 

57 numSigmaClip = pexConfig.Field( 

58 dtype=float, 

59 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

60 default=3.0, 

61 ) 

62 overscanIsInt = pexConfig.Field( 

63 dtype=bool, 

64 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN" + 

65 " and fitType=MEDIAN_PER_ROW.", 

66 default=True, 

67 ) 

68 

69 

70class OverscanCorrectionTask(pipeBase.Task): 

71 """Correction task for overscan. 

72 

73 This class contains a number of utilities that are easier to 

74 understand and use when they are not embedded in nested if/else 

75 loops. 

76 

77 Parameters 

78 ---------- 

79 statControl : `lsst.afw.math.StatisticsControl`, optional 

80 Statistics control object. 

81 """ 

82 ConfigClass = OverscanCorrectionTaskConfig 

83 _DefaultName = "overscan" 

84 

85 def __init__(self, statControl=None, **kwargs): 

86 super().__init__(**kwargs) 

87 if statControl: 

88 self.statControl = statControl 

89 else: 

90 self.statControl = afwMath.StatisticsControl() 

91 self.statControl.setNumSigmaClip(self.config.numSigmaClip) 

92 

93 def run(self, ampImage, overscanImage): 

94 """Measure and remove an overscan from an amplifier image. 

95 

96 Parameters 

97 ---------- 

98 ampImage : `lsst.afw.image.Image` 

99 Image data that will have the overscan removed. 

100 overscanImage : `lsst.afw.image.Image` 

101 Overscan data that the overscan is measured from. 

102 

103 Returns 

104 ------- 

105 overscanResults : `lsst.pipe.base.Struct` 

106 Result struct with components: 

107 

108 ``imageFit`` 

109 Value or fit subtracted from the amplifier image data 

110 (scalar or `lsst.afw.image.Image`). 

111 ``overscanFit`` 

112 Value or fit subtracted from the overscan image data 

113 (scalar or `lsst.afw.image.Image`). 

114 ``overscanImage`` 

115 Image of the overscan region with the overscan 

116 correction applied (`lsst.afw.image.Image`). This 

117 quantity is used to estimate the amplifier read noise 

118 empirically. 

119 

120 Raises 

121 ------ 

122 RuntimeError 

123 Raised if an invalid overscan type is set. 

124 

125 """ 

126 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'): 

127 overscanResult = self.measureConstantOverscan(overscanImage) 

128 overscanValue = overscanResult.overscanValue 

129 offImage = overscanValue 

130 overscanModel = overscanValue 

131 maskSuspect = None 

132 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG', 

133 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'): 

134 overscanResult = self.measureVectorOverscan(overscanImage) 

135 overscanValue = overscanResult.overscanValue 

136 maskArray = overscanResult.maskArray 

137 isTransposed = overscanResult.isTransposed 

138 

139 offImage = afwImage.ImageF(ampImage.getDimensions()) 

140 offArray = offImage.getArray() 

141 overscanModel = afwImage.ImageF(overscanImage.getDimensions()) 

142 overscanArray = overscanModel.getArray() 

143 

144 if hasattr(ampImage, 'getMask'): 

145 maskSuspect = afwImage.Mask(ampImage.getDimensions()) 

146 else: 

147 maskSuspect = None 

148 

149 if isTransposed: 

150 offArray[:, :] = overscanValue[np.newaxis, :] 

151 overscanArray[:, :] = overscanValue[np.newaxis, :] 

152 if maskSuspect: 

153 maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT") 

154 else: 

155 offArray[:, :] = overscanValue[:, np.newaxis] 

156 overscanArray[:, :] = overscanValue[:, np.newaxis] 

157 if maskSuspect: 

158 maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT") 

159 else: 

160 raise RuntimeError('%s : %s an invalid overscan type' % 

161 ("overscanCorrection", self.config.fitType)) 

162 

163 self.debugView(overscanImage, overscanValue) 

164 

165 ampImage -= offImage 

166 if maskSuspect: 

167 ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :] 

168 overscanImage -= overscanModel 

169 return pipeBase.Struct(imageFit=offImage, 

170 overscanFit=overscanModel, 

171 overscanImage=overscanImage, 

172 edgeMask=maskSuspect) 

173 

174 @staticmethod 

175 def integerConvert(image): 

176 """Return an integer version of the input image. 

177 

178 Parameters 

179 ---------- 

180 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

181 Image to convert to integers. 

182 

183 Returns 

184 ------- 

185 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

186 The integer converted image. 

187 

188 Raises 

189 ------ 

190 RuntimeError 

191 Raised if the input image could not be converted. 

192 """ 

193 if hasattr(image, "image"): 

194 # Is a maskedImage: 

195 imageI = image.image.convertI() 

196 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance) 

197 elif hasattr(image, "convertI"): 

198 # Is an Image: 

199 outI = image.convertI() 

200 elif hasattr(image, "astype"): 

201 # Is a numpy array: 

202 outI = image.astype(int) 

203 else: 

204 raise RuntimeError("Could not convert this to integers: %s %s %s", 

205 image, type(image), dir(image)) 

206 return outI 

207 

208 # Constant methods 

209 def measureConstantOverscan(self, image): 

210 """Measure a constant overscan value. 

211 

212 Parameters 

213 ---------- 

214 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

215 Image data to measure the overscan from. 

216 

217 Returns 

218 ------- 

219 results : `lsst.pipe.base.Struct` 

220 Overscan result with entries: 

221 - ``overscanValue``: Overscan value to subtract (`float`) 

222 - ``maskArray``: Placeholder for a mask array (`list`) 

223 - ``isTransposed``: Orientation of the overscan (`bool`) 

224 """ 

225 if self.config.fitType == 'MEDIAN': 

226 calcImage = self.integerConvert(image) 

227 else: 

228 calcImage = image 

229 

230 fitType = afwMath.stringToStatisticsProperty(self.config.fitType) 

231 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue() 

232 

233 return pipeBase.Struct(overscanValue=overscanValue, 

234 maskArray=None, 

235 isTransposed=False) 

236 

237 # Vector correction utilities 

238 @staticmethod 

239 def getImageArray(image): 

240 """Extract the numpy array from the input image. 

241 

242 Parameters 

243 ---------- 

244 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

245 Image data to pull array from. 

246 

247 calcImage : `numpy.ndarray` 

248 Image data array for numpy operating. 

249 """ 

250 if hasattr(image, "getImage"): 

251 calcImage = image.getImage().getArray() 

252 else: 

253 calcImage = image.getArray() 

254 return calcImage 

255 

256 @staticmethod 

257 def transpose(imageArray): 

258 """Transpose input numpy array if necessary. 

259 

260 Parameters 

261 ---------- 

262 imageArray : `numpy.ndarray` 

263 Image data to transpose. 

264 

265 Returns 

266 ------- 

267 imageArray : `numpy.ndarray` 

268 Transposed image data. 

269 isTransposed : `bool` 

270 Indicates whether the input data was transposed. 

271 """ 

272 if np.argmin(imageArray.shape) == 0: 

273 return np.transpose(imageArray), True 

274 else: 

275 return imageArray, False 

276 

277 def maskOutliers(self, imageArray): 

278 """Mask outliers in a row of overscan data from a robust sigma 

279 clipping procedure. 

280 

281 Parameters 

282 ---------- 

283 imageArray : `numpy.ndarray` 

284 Image to filter along numpy axis=1. 

285 

286 Returns 

287 ------- 

288 maskedArray : `numpy.ma.masked_array` 

289 Masked image marking outliers. 

290 """ 

291 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1) 

292 axisMedians = median 

293 axisStdev = 0.74*(uq - lq) # robust stdev 

294 

295 diff = np.abs(imageArray - axisMedians[:, np.newaxis]) 

296 return np.ma.masked_where(diff > self.statControl.getNumSigmaClip() * 

297 axisStdev[:, np.newaxis], imageArray) 

298 

299 @staticmethod 

300 def collapseArray(maskedArray): 

301 """Collapse overscan array (and mask) to a 1-D vector of values. 

302 

303 Parameters 

304 ---------- 

305 maskedArray : `numpy.ma.masked_array` 

306 Masked array of input overscan data. 

307 

308 Returns 

309 ------- 

310 collapsed : `numpy.ma.masked_array` 

311 Single dimensional overscan data, combined with the mean. 

312 """ 

313 collapsed = np.mean(maskedArray, axis=1) 

314 if collapsed.mask.sum() > 0: 

315 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1) 

316 return collapsed 

317 

318 def collapseArrayMedian(self, maskedArray): 

319 """Collapse overscan array (and mask) to a 1-D vector of using the 

320 correct integer median of row-values. 

321 

322 Parameters 

323 ---------- 

324 maskedArray : `numpy.ma.masked_array` 

325 Masked array of input overscan data. 

326 

327 Returns 

328 ------- 

329 collapsed : `numpy.ma.masked_array` 

330 Single dimensional overscan data, combined with the afwMath median. 

331 """ 

332 integerMI = self.integerConvert(maskedArray) 

333 

334 collapsed = [] 

335 fitType = afwMath.stringToStatisticsProperty('MEDIAN') 

336 for row in integerMI: 

337 rowMedian = afwMath.makeStatistics(row, fitType, self.statControl).getValue() 

338 collapsed.append(rowMedian) 

339 

340 return np.array(collapsed) 

341 

342 def splineFit(self, indices, collapsed, numBins): 

343 """Wrapper function to match spline fit API to polynomial fit API. 

344 

345 Parameters 

346 ---------- 

347 indices : `numpy.ndarray` 

348 Locations to evaluate the spline. 

349 collapsed : `numpy.ndarray` 

350 Collapsed overscan values corresponding to the spline 

351 evaluation points. 

352 numBins : `int` 

353 Number of bins to use in constructing the spline. 

354 

355 Returns 

356 ------- 

357 interp : `lsst.afw.math.Interpolate` 

358 Interpolation object for later evaluation. 

359 """ 

360 if not np.ma.is_masked(collapsed): 

361 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask]) 

362 

363 numPerBin, binEdges = np.histogram(indices, bins=numBins, 

364 weights=1 - collapsed.mask.astype(int)) 

365 with np.errstate(invalid="ignore"): 

366 values = np.histogram(indices, bins=numBins, 

367 weights=collapsed.data*~collapsed.mask)[0]/numPerBin 

368 binCenters = np.histogram(indices, bins=numBins, 

369 weights=indices*~collapsed.mask)[0]/numPerBin 

370 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0], 

371 values.astype(float)[numPerBin > 0], 

372 afwMath.stringToInterpStyle(self.config.fitType)) 

373 return interp 

374 

375 @staticmethod 

376 def splineEval(indices, interp): 

377 """Wrapper function to match spline evaluation API to polynomial fit API. 

378 

379 Parameters 

380 ---------- 

381 indices : `numpy.ndarray` 

382 Locations to evaluate the spline. 

383 interp : `lsst.afw.math.interpolate` 

384 Interpolation object to use. 

385 

386 Returns 

387 ------- 

388 values : `numpy.ndarray` 

389 Evaluated spline values at each index. 

390 """ 

391 

392 return interp.interpolate(indices.astype(float)) 

393 

394 @staticmethod 

395 def maskExtrapolated(collapsed): 

396 """Create mask if edges are extrapolated. 

397 

398 Parameters 

399 ---------- 

400 collapsed : `numpy.ma.masked_array` 

401 Masked array to check the edges of. 

402 

403 Returns 

404 ------- 

405 maskArray : `numpy.ndarray` 

406 Boolean numpy array of pixels to mask. 

407 """ 

408 maskArray = np.full_like(collapsed, False, dtype=bool) 

409 if np.ma.is_masked(collapsed): 

410 num = len(collapsed) 

411 for low in range(num): 

412 if not collapsed.mask[low]: 

413 break 

414 if low > 0: 

415 maskArray[:low] = True 

416 for high in range(1, num): 

417 if not collapsed.mask[-high]: 

418 break 

419 if high > 1: 

420 maskArray[-high:] = True 

421 return maskArray 

422 

423 def measureVectorOverscan(self, image): 

424 """Calculate the 1-d vector overscan from the input overscan image. 

425 

426 Parameters 

427 ---------- 

428 image : `lsst.afw.image.MaskedImage` 

429 Image containing the overscan data. 

430 

431 Returns 

432 ------- 

433 results : `lsst.pipe.base.Struct` 

434 Overscan result with entries: 

435 - ``overscanValue``: Overscan value to subtract (`float`) 

436 - ``maskArray`` : `list` [ `bool` ] 

437 List of rows that should be masked as ``SUSPECT`` when the 

438 overscan solution is applied. 

439 - ``isTransposed`` : `bool` 

440 Indicates if the overscan data was transposed during 

441 calcuation, noting along which axis the overscan should be 

442 subtracted. 

443 """ 

444 calcImage = self.getImageArray(image) 

445 

446 # operate on numpy-arrays from here 

447 calcImage, isTransposed = self.transpose(calcImage) 

448 masked = self.maskOutliers(calcImage) 

449 

450 if self.config.fitType == 'MEDIAN_PER_ROW': 

451 overscanVector = self.collapseArrayMedian(masked) 

452 maskArray = self.maskExtrapolated(overscanVector) 

453 else: 

454 collapsed = self.collapseArray(masked) 

455 

456 num = len(collapsed) 

457 indices = 2.0*np.arange(num)/float(num) - 1.0 

458 

459 poly = np.polynomial 

460 fitter, evaler = { 

461 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval), 

462 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval), 

463 'LEG': (poly.legendre.legfit, poly.legendre.legval), 

464 'NATURAL_SPLINE': (self.splineFit, self.splineEval), 

465 'CUBIC_SPLINE': (self.splineFit, self.splineEval), 

466 'AKIMA_SPLINE': (self.splineFit, self.splineEval) 

467 }[self.config.fitType] 

468 

469 coeffs = fitter(indices, collapsed, self.config.order) 

470 overscanVector = evaler(indices, coeffs) 

471 maskArray = self.maskExtrapolated(collapsed) 

472 return pipeBase.Struct(overscanValue=np.array(overscanVector), 

473 maskArray=maskArray, 

474 isTransposed=isTransposed) 

475 

476 def debugView(self, image, model): 

477 """Debug display for the final overscan solution. 

478 

479 Parameters 

480 ---------- 

481 image : `lsst.afw.image.Image` 

482 Input image the overscan solution was determined from. 

483 model : `numpy.ndarray` or `float` 

484 Overscan model determined for the image. 

485 """ 

486 import lsstDebug 

487 if not lsstDebug.Info(__name__).display: 

488 return 

489 

490 calcImage = self.getImageArray(image) 

491 calcImage, isTransposed = self.transpose(calcImage) 

492 masked = self.maskOutliers(calcImage) 

493 collapsed = self.collapseArray(masked) 

494 

495 num = len(collapsed) 

496 indices = 2.0 * np.arange(num)/float(num) - 1.0 

497 

498 if np.ma.is_masked(collapsed): 

499 collapsedMask = collapsed.mask 

500 else: 

501 collapsedMask = np.array(num*[np.ma.nomask]) 

502 

503 import matplotlib.pyplot as plot 

504 figure = plot.figure(1) 

505 figure.clear() 

506 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

507 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+') 

508 if collapsedMask.sum() > 0: 

509 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+') 

510 if isinstance(model, np.ndarray): 

511 plotModel = model 

512 else: 

513 plotModel = np.zeros_like(indices) 

514 plotModel += model 

515 axes.plot(indices, plotModel, 'r-') 

516 plot.xlabel("centered/scaled position along overscan region") 

517 plot.ylabel("pixel value/fit value") 

518 figure.show() 

519 prompt = "Press Enter or c to continue [chp]..." 

520 while True: 

521 ans = input(prompt).lower() 

522 if ans in ("", " ", "c",): 

523 break 

524 elif ans in ("p", ): 

525 import pdb 

526 pdb.set_trace() 

527 elif ans in ("h", ): 

528 print("[h]elp [c]ontinue [p]db") 

529 plot.close()