Coverage for python/lsst/ip/isr/overscan.py: 13%

210 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-26 10:32 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import lsst.afw.math as afwMath 

24import lsst.afw.image as afwImage 

25import lsst.pipe.base as pipeBase 

26import lsst.pex.config as pexConfig 

27 

28__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"] 

29 

30 

31class OverscanCorrectionTaskConfig(pexConfig.Config): 

32 """Overscan correction options. 

33 """ 

34 fitType = pexConfig.ChoiceField( 

35 dtype=str, 

36 doc="The method for fitting the overscan bias level.", 

37 default='MEDIAN', 

38 allowed={ 

39 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

40 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

41 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

42 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

43 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

44 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

45 "MEAN": "Correct using the mean of the overscan region", 

46 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

47 "MEDIAN": "Correct using the median of the overscan region", 

48 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

49 }, 

50 ) 

51 order = pexConfig.Field( 

52 dtype=int, 

53 doc=("Order of polynomial to fit if overscan fit type is a polynomial, " 

54 "or number of spline knots if overscan fit type is a spline."), 

55 default=1, 

56 ) 

57 numSigmaClip = pexConfig.Field( 

58 dtype=float, 

59 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

60 default=3.0, 

61 ) 

62 maskPlanes = pexConfig.ListField( 

63 dtype=str, 

64 doc="Mask planes to reject when measuring overscan", 

65 default=['SAT'], 

66 ) 

67 overscanIsInt = pexConfig.Field( 

68 dtype=bool, 

69 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN" 

70 " and fitType=MEDIAN_PER_ROW.", 

71 default=True, 

72 ) 

73 

74 

75class OverscanCorrectionTask(pipeBase.Task): 

76 """Correction task for overscan. 

77 

78 This class contains a number of utilities that are easier to 

79 understand and use when they are not embedded in nested if/else 

80 loops. 

81 

82 Parameters 

83 ---------- 

84 statControl : `lsst.afw.math.StatisticsControl`, optional 

85 Statistics control object. 

86 """ 

87 ConfigClass = OverscanCorrectionTaskConfig 

88 _DefaultName = "overscan" 

89 

90 def __init__(self, statControl=None, **kwargs): 

91 super().__init__(**kwargs) 

92 self.allowDebug = True 

93 

94 if statControl: 

95 self.statControl = statControl 

96 else: 

97 self.statControl = afwMath.StatisticsControl() 

98 self.statControl.setNumSigmaClip(self.config.numSigmaClip) 

99 self.statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes)) 

100 

101 def run(self, ampImage, overscanImage, amp=None): 

102 """Measure and remove an overscan from an amplifier image. 

103 

104 Parameters 

105 ---------- 

106 ampImage : `lsst.afw.image.Image` 

107 Image data that will have the overscan removed. 

108 overscanImage : `lsst.afw.image.Image` 

109 Overscan data that the overscan is measured from. 

110 amp : `lsst.afw.cameraGeom.Amplifier`, optional 

111 Amplifier to use for debugging purposes. 

112 

113 Returns 

114 ------- 

115 overscanResults : `lsst.pipe.base.Struct` 

116 Result struct with components: 

117 

118 ``imageFit`` 

119 Value or fit subtracted from the amplifier image data 

120 (scalar or `lsst.afw.image.Image`). 

121 ``overscanFit`` 

122 Value or fit subtracted from the overscan image data 

123 (scalar or `lsst.afw.image.Image`). 

124 ``overscanImage`` 

125 Image of the overscan region with the overscan 

126 correction applied (`lsst.afw.image.Image`). This 

127 quantity is used to estimate the amplifier read noise 

128 empirically. 

129 

130 Raises 

131 ------ 

132 RuntimeError 

133 Raised if an invalid overscan type is set. 

134 

135 """ 

136 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'): 

137 overscanResult = self.measureConstantOverscan(overscanImage) 

138 overscanValue = overscanResult.overscanValue 

139 offImage = overscanValue 

140 overscanModel = overscanValue 

141 maskSuspect = None 

142 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG', 

143 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'): 

144 overscanResult = self.measureVectorOverscan(overscanImage) 

145 overscanValue = overscanResult.overscanValue 

146 maskArray = overscanResult.maskArray 

147 isTransposed = overscanResult.isTransposed 

148 

149 offImage = afwImage.ImageF(ampImage.getDimensions()) 

150 offArray = offImage.getArray() 

151 overscanModel = afwImage.ImageF(overscanImage.getDimensions()) 

152 overscanArray = overscanModel.getArray() 

153 

154 if hasattr(ampImage, 'getMask'): 

155 maskSuspect = afwImage.Mask(ampImage.getDimensions()) 

156 else: 

157 maskSuspect = None 

158 

159 if isTransposed: 

160 offArray[:, :] = overscanValue[np.newaxis, :] 

161 overscanArray[:, :] = overscanValue[np.newaxis, :] 

162 if maskSuspect: 

163 maskSuspect.getArray()[:, maskArray] |= ampImage.getMask().getPlaneBitMask("SUSPECT") 

164 else: 

165 offArray[:, :] = overscanValue[:, np.newaxis] 

166 overscanArray[:, :] = overscanValue[:, np.newaxis] 

167 if maskSuspect: 

168 maskSuspect.getArray()[maskArray, :] |= ampImage.getMask().getPlaneBitMask("SUSPECT") 

169 else: 

170 raise RuntimeError('%s : %s an invalid overscan type' % 

171 ("overscanCorrection", self.config.fitType)) 

172 

173 self.debugView(overscanImage, overscanValue, amp) 

174 

175 ampImage -= offImage 

176 if maskSuspect: 

177 ampImage.getMask().getArray()[:, :] |= maskSuspect.getArray()[:, :] 

178 overscanImage -= overscanModel 

179 return pipeBase.Struct(imageFit=offImage, 

180 overscanFit=overscanModel, 

181 overscanImage=overscanImage, 

182 edgeMask=maskSuspect) 

183 

184 @staticmethod 

185 def integerConvert(image): 

186 """Return an integer version of the input image. 

187 

188 Parameters 

189 ---------- 

190 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

191 Image to convert to integers. 

192 

193 Returns 

194 ------- 

195 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

196 The integer converted image. 

197 

198 Raises 

199 ------ 

200 RuntimeError 

201 Raised if the input image could not be converted. 

202 """ 

203 if hasattr(image, "image"): 

204 # Is a maskedImage: 

205 imageI = image.image.convertI() 

206 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance) 

207 elif hasattr(image, "convertI"): 

208 # Is an Image: 

209 outI = image.convertI() 

210 elif hasattr(image, "astype"): 

211 # Is a numpy array: 

212 outI = image.astype(int) 

213 else: 

214 raise RuntimeError("Could not convert this to integers: %s %s %s", 

215 image, type(image), dir(image)) 

216 return outI 

217 

218 # Constant methods 

219 def measureConstantOverscan(self, image): 

220 """Measure a constant overscan value. 

221 

222 Parameters 

223 ---------- 

224 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

225 Image data to measure the overscan from. 

226 

227 Returns 

228 ------- 

229 results : `lsst.pipe.base.Struct` 

230 Overscan result with entries: 

231 - ``overscanValue``: Overscan value to subtract (`float`) 

232 - ``maskArray``: Placeholder for a mask array (`list`) 

233 - ``isTransposed``: Orientation of the overscan (`bool`) 

234 """ 

235 if self.config.fitType == 'MEDIAN': 

236 calcImage = self.integerConvert(image) 

237 else: 

238 calcImage = image 

239 

240 fitType = afwMath.stringToStatisticsProperty(self.config.fitType) 

241 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue() 

242 

243 return pipeBase.Struct(overscanValue=overscanValue, 

244 maskArray=None, 

245 isTransposed=False) 

246 

247 # Vector correction utilities 

248 def getImageArray(self, image): 

249 """Extract the numpy array from the input image. 

250 

251 Parameters 

252 ---------- 

253 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

254 Image data to pull array from. 

255 

256 calcImage : `numpy.ndarray` 

257 Image data array for numpy operating. 

258 """ 

259 if hasattr(image, "getImage"): 

260 calcImage = image.getImage().getArray() 

261 calcImage = np.ma.masked_where(image.getMask().getArray() & self.statControl.getAndMask(), 

262 calcImage) 

263 else: 

264 calcImage = image.getArray() 

265 return calcImage 

266 

267 @staticmethod 

268 def transpose(imageArray): 

269 """Transpose input numpy array if necessary. 

270 

271 Parameters 

272 ---------- 

273 imageArray : `numpy.ndarray` 

274 Image data to transpose. 

275 

276 Returns 

277 ------- 

278 imageArray : `numpy.ndarray` 

279 Transposed image data. 

280 isTransposed : `bool` 

281 Indicates whether the input data was transposed. 

282 """ 

283 if np.argmin(imageArray.shape) == 0: 

284 return np.transpose(imageArray), True 

285 else: 

286 return imageArray, False 

287 

288 def maskOutliers(self, imageArray): 

289 """Mask outliers in a row of overscan data from a robust sigma 

290 clipping procedure. 

291 

292 Parameters 

293 ---------- 

294 imageArray : `numpy.ndarray` 

295 Image to filter along numpy axis=1. 

296 

297 Returns 

298 ------- 

299 maskedArray : `numpy.ma.masked_array` 

300 Masked image marking outliers. 

301 """ 

302 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1) 

303 axisMedians = median 

304 axisStdev = 0.74*(uq - lq) # robust stdev 

305 

306 diff = np.abs(imageArray - axisMedians[:, np.newaxis]) 

307 return np.ma.masked_where(diff > self.statControl.getNumSigmaClip() 

308 * axisStdev[:, np.newaxis], imageArray) 

309 

310 @staticmethod 

311 def collapseArray(maskedArray): 

312 """Collapse overscan array (and mask) to a 1-D vector of values. 

313 

314 Parameters 

315 ---------- 

316 maskedArray : `numpy.ma.masked_array` 

317 Masked array of input overscan data. 

318 

319 Returns 

320 ------- 

321 collapsed : `numpy.ma.masked_array` 

322 Single dimensional overscan data, combined with the mean. 

323 """ 

324 collapsed = np.mean(maskedArray, axis=1) 

325 if collapsed.mask.sum() > 0: 

326 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1) 

327 return collapsed 

328 

329 def collapseArrayMedian(self, maskedArray): 

330 """Collapse overscan array (and mask) to a 1-D vector of using the 

331 correct integer median of row-values. 

332 

333 Parameters 

334 ---------- 

335 maskedArray : `numpy.ma.masked_array` 

336 Masked array of input overscan data. 

337 

338 Returns 

339 ------- 

340 collapsed : `numpy.ma.masked_array` 

341 Single dimensional overscan data, combined with the afwMath median. 

342 """ 

343 integerMI = self.integerConvert(maskedArray) 

344 

345 collapsed = [] 

346 fitType = afwMath.stringToStatisticsProperty('MEDIAN') 

347 for row in integerMI: 

348 newRow = row.compressed() 

349 if len(newRow) > 0: 

350 rowMedian = afwMath.makeStatistics(newRow, fitType, self.statControl).getValue() 

351 else: 

352 rowMedian = np.nan 

353 collapsed.append(rowMedian) 

354 

355 return np.array(collapsed) 

356 

357 def splineFit(self, indices, collapsed, numBins): 

358 """Wrapper function to match spline fit API to polynomial fit API. 

359 

360 Parameters 

361 ---------- 

362 indices : `numpy.ndarray` 

363 Locations to evaluate the spline. 

364 collapsed : `numpy.ndarray` 

365 Collapsed overscan values corresponding to the spline 

366 evaluation points. 

367 numBins : `int` 

368 Number of bins to use in constructing the spline. 

369 

370 Returns 

371 ------- 

372 interp : `lsst.afw.math.Interpolate` 

373 Interpolation object for later evaluation. 

374 """ 

375 if not np.ma.is_masked(collapsed): 

376 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask]) 

377 

378 numPerBin, binEdges = np.histogram(indices, bins=numBins, 

379 weights=1 - collapsed.mask.astype(int)) 

380 with np.errstate(invalid="ignore"): 

381 values = np.histogram(indices, bins=numBins, 

382 weights=collapsed.data*~collapsed.mask)[0]/numPerBin 

383 binCenters = np.histogram(indices, bins=numBins, 

384 weights=indices*~collapsed.mask)[0]/numPerBin 

385 

386 if len(binCenters[numPerBin > 0]) < 5: 

387 self.log.warn("Cannot do spline fitting for overscan: %s valid points.", 

388 len(binCenters[numPerBin > 0])) 

389 # Return a scalar value if we have one, otherwise 

390 # return zero. This amplifier is hopefully already 

391 # masked. 

392 if len(values[numPerBin > 0]) != 0: 

393 return float(values[numPerBin > 0][0]) 

394 else: 

395 return 0.0 

396 

397 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0], 

398 values.astype(float)[numPerBin > 0], 

399 afwMath.stringToInterpStyle(self.config.fitType)) 

400 return interp 

401 

402 @staticmethod 

403 def splineEval(indices, interp): 

404 """Wrapper function to match spline evaluation API to polynomial fit 

405 API. 

406 

407 Parameters 

408 ---------- 

409 indices : `numpy.ndarray` 

410 Locations to evaluate the spline. 

411 interp : `lsst.afw.math.interpolate` 

412 Interpolation object to use. 

413 

414 Returns 

415 ------- 

416 values : `numpy.ndarray` 

417 Evaluated spline values at each index. 

418 """ 

419 

420 return interp.interpolate(indices.astype(float)) 

421 

422 @staticmethod 

423 def maskExtrapolated(collapsed): 

424 """Create mask if edges are extrapolated. 

425 

426 Parameters 

427 ---------- 

428 collapsed : `numpy.ma.masked_array` 

429 Masked array to check the edges of. 

430 

431 Returns 

432 ------- 

433 maskArray : `numpy.ndarray` 

434 Boolean numpy array of pixels to mask. 

435 """ 

436 maskArray = np.full_like(collapsed, False, dtype=bool) 

437 if np.ma.is_masked(collapsed): 

438 num = len(collapsed) 

439 for low in range(num): 

440 if not collapsed.mask[low]: 

441 break 

442 if low > 0: 

443 maskArray[:low] = True 

444 for high in range(1, num): 

445 if not collapsed.mask[-high]: 

446 break 

447 if high > 1: 

448 maskArray[-high:] = True 

449 return maskArray 

450 

451 def measureVectorOverscan(self, image): 

452 """Calculate the 1-d vector overscan from the input overscan image. 

453 

454 Parameters 

455 ---------- 

456 image : `lsst.afw.image.MaskedImage` 

457 Image containing the overscan data. 

458 

459 Returns 

460 ------- 

461 results : `lsst.pipe.base.Struct` 

462 Overscan result with entries: 

463 - ``overscanValue``: Overscan value to subtract (`float`) 

464 - ``maskArray`` : `list` [ `bool` ] 

465 List of rows that should be masked as ``SUSPECT`` when the 

466 overscan solution is applied. 

467 - ``isTransposed`` : `bool` 

468 Indicates if the overscan data was transposed during 

469 calcuation, noting along which axis the overscan should be 

470 subtracted. 

471 """ 

472 calcImage = self.getImageArray(image) 

473 

474 # operate on numpy-arrays from here 

475 calcImage, isTransposed = self.transpose(calcImage) 

476 masked = self.maskOutliers(calcImage) 

477 

478 if self.config.fitType == 'MEDIAN_PER_ROW': 

479 overscanVector = self.collapseArrayMedian(masked) 

480 maskArray = self.maskExtrapolated(overscanVector) 

481 else: 

482 collapsed = self.collapseArray(masked) 

483 

484 num = len(collapsed) 

485 indices = 2.0*np.arange(num)/float(num) - 1.0 

486 

487 poly = np.polynomial 

488 fitter, evaler = { 

489 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval), 

490 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval), 

491 'LEG': (poly.legendre.legfit, poly.legendre.legval), 

492 'NATURAL_SPLINE': (self.splineFit, self.splineEval), 

493 'CUBIC_SPLINE': (self.splineFit, self.splineEval), 

494 'AKIMA_SPLINE': (self.splineFit, self.splineEval) 

495 }[self.config.fitType] 

496 

497 # These are the polynomial coefficients, or an 

498 # interpolation object. 

499 coeffs = fitter(indices, collapsed, self.config.order) 

500 

501 if isinstance(coeffs, float): 

502 self.log.warn("Using fallback value %f due to fitter failure. Amplifier will be masked.", 

503 coeffs) 

504 overscanVector = np.full_like(indices, coeffs) 

505 maskArray = np.full_like(collapsed, True, dtype=bool) 

506 else: 

507 # Otherwise we can just use things as normal. 

508 overscanVector = evaler(indices, coeffs) 

509 maskArray = self.maskExtrapolated(collapsed) 

510 return pipeBase.Struct(overscanValue=np.array(overscanVector), 

511 maskArray=maskArray, 

512 isTransposed=isTransposed) 

513 

514 def debugView(self, image, model, amp=None): 

515 """Debug display for the final overscan solution. 

516 

517 Parameters 

518 ---------- 

519 image : `lsst.afw.image.Image` 

520 Input image the overscan solution was determined from. 

521 model : `numpy.ndarray` or `float` 

522 Overscan model determined for the image. 

523 amp : `lsst.afw.cameraGeom.Amplifier`, optional 

524 Amplifier to extract diagnostic information. 

525 """ 

526 import lsstDebug 

527 if not lsstDebug.Info(__name__).display: 

528 return 

529 if not self.allowDebug: 

530 return 

531 

532 calcImage = self.getImageArray(image) 

533 calcImage, isTransposed = self.transpose(calcImage) 

534 masked = self.maskOutliers(calcImage) 

535 collapsed = self.collapseArray(masked) 

536 

537 num = len(collapsed) 

538 indices = 2.0 * np.arange(num)/float(num) - 1.0 

539 

540 if np.ma.is_masked(collapsed): 

541 collapsedMask = collapsed.mask 

542 else: 

543 collapsedMask = np.array(num*[np.ma.nomask]) 

544 

545 import matplotlib.pyplot as plot 

546 figure = plot.figure(1) 

547 figure.clear() 

548 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

549 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+') 

550 if collapsedMask.sum() > 0: 

551 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+') 

552 if isinstance(model, np.ndarray): 

553 plotModel = model 

554 else: 

555 plotModel = np.zeros_like(indices) 

556 plotModel += model 

557 axes.plot(indices, plotModel, 'r-') 

558 plot.xlabel("centered/scaled position along overscan region") 

559 plot.ylabel("pixel value/fit value") 

560 if amp: 

561 plot.title(f"{amp.getName()} DataX: " 

562 f"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]" 

563 f"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:" 

564 f"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}") 

565 else: 

566 plot.title("No amp supplied.") 

567 figure.show() 

568 prompt = "Press Enter or c to continue [chp]..." 

569 while True: 

570 ans = input(prompt).lower() 

571 if ans in ("", " ", "c",): 

572 break 

573 elif ans in ("p", ): 

574 import pdb 

575 pdb.set_trace() 

576 elif ans in ('x', ): 

577 self.allowDebug = False 

578 break 

579 elif ans in ("h", ): 

580 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

581 plot.close()