Coverage for python/lsst/ip/isr/overscan.py: 12%

299 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-14 03:17 -0800

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["OverscanCorrectionTaskConfig", "OverscanCorrectionTask"] 

23 

24import numpy as np 

25import lsst.afw.math as afwMath 

26import lsst.afw.image as afwImage 

27import lsst.geom as geom 

28import lsst.pipe.base as pipeBase 

29import lsst.pex.config as pexConfig 

30 

31from .isr import fitOverscanImage 

32from .isrFunctions import makeThresholdMask, countMaskedPixels 

33 

34 

35class OverscanCorrectionTaskConfig(pexConfig.Config): 

36 """Overscan correction options. 

37 """ 

38 fitType = pexConfig.ChoiceField( 

39 dtype=str, 

40 doc="The method for fitting the overscan bias level.", 

41 default='MEDIAN', 

42 allowed={ 

43 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

44 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

45 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

46 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

47 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

48 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

49 "MEAN": "Correct using the mean of the overscan region", 

50 "MEANCLIP": "Correct using a clipped mean of the overscan region", 

51 "MEDIAN": "Correct using the median of the overscan region", 

52 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region", 

53 }, 

54 ) 

55 order = pexConfig.Field( 

56 dtype=int, 

57 doc=("Order of polynomial to fit if overscan fit type is a polynomial, " 

58 "or number of spline knots if overscan fit type is a spline."), 

59 default=1, 

60 ) 

61 numSigmaClip = pexConfig.Field( 

62 dtype=float, 

63 doc="Rejection threshold (sigma) for collapsing overscan before fit", 

64 default=3.0, 

65 ) 

66 maskPlanes = pexConfig.ListField( 

67 dtype=str, 

68 doc="Mask planes to reject when measuring overscan", 

69 default=['BAD', 'SAT'], 

70 ) 

71 overscanIsInt = pexConfig.Field( 

72 dtype=bool, 

73 doc="Treat overscan as an integer image for purposes of fitType=MEDIAN" 

74 " and fitType=MEDIAN_PER_ROW.", 

75 default=True, 

76 ) 

77 

78 doParallelOverscan = pexConfig.Field( 

79 dtype=bool, 

80 doc="Correct using parallel overscan after serial overscan correction?", 

81 default=False, 

82 ) 

83 parallelOverscanMaskThreshold = pexConfig.RangeField( 

84 dtype=float, 

85 doc="Minimum fraction of pixels in parallel overscan region necessary " 

86 "for parallel overcan correction.", 

87 default=0.1, 

88 min=0.0, 

89 max=1.0, 

90 inclusiveMin=True, 

91 inclusiveMax=True, 

92 ) 

93 

94 leadingColumnsToSkip = pexConfig.Field( 

95 dtype=int, 

96 doc="Number of leading columns to skip in serial overscan correction.", 

97 default=0, 

98 ) 

99 trailingColumnsToSkip = pexConfig.Field( 

100 dtype=int, 

101 doc="Number of trailing columns to skip in serial overscan correction.", 

102 default=0, 

103 ) 

104 leadingRowsToSkip = pexConfig.Field( 

105 dtype=int, 

106 doc="Number of leading rows to skip in parallel overscan correction.", 

107 default=0, 

108 ) 

109 trailingRowsToSkip = pexConfig.Field( 

110 dtype=int, 

111 doc="Number of trailing rows to skip in parallel overscan correction.", 

112 default=0, 

113 ) 

114 

115 maxDeviation = pexConfig.Field( 115 ↛ exitline 115 didn't jump to the function exit

116 dtype=float, 

117 doc="Maximum deviation from median (in ADU) to mask in overscan correction.", 

118 default=1000.0, check=lambda x: x > 0, 

119 ) 

120 

121 

122class OverscanCorrectionTask(pipeBase.Task): 

123 """Correction task for overscan. 

124 

125 This class contains a number of utilities that are easier to 

126 understand and use when they are not embedded in nested if/else 

127 loops. 

128 

129 Parameters 

130 ---------- 

131 statControl : `lsst.afw.math.StatisticsControl`, optional 

132 Statistics control object. 

133 """ 

134 ConfigClass = OverscanCorrectionTaskConfig 

135 _DefaultName = "overscan" 

136 

137 def __init__(self, statControl=None, **kwargs): 

138 super().__init__(**kwargs) 

139 self.allowDebug = True 

140 

141 if statControl: 

142 self.statControl = statControl 

143 else: 

144 self.statControl = afwMath.StatisticsControl() 

145 self.statControl.setNumSigmaClip(self.config.numSigmaClip) 

146 self.statControl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.maskPlanes)) 

147 

148 def run(self, exposure, amp, isTransposed=False): 

149 """Measure and remove an overscan from an amplifier image. 

150 

151 Parameters 

152 ---------- 

153 exposure : `lsst.afw.image.Exposure` 

154 Image data that will have the overscan corrections applied. 

155 amp : `lsst.afw.cameraGeom.Amplifier` 

156 Amplifier to use for debugging purposes. 

157 isTransposed : `bool`, optional 

158 Is the image transposed, such that serial and parallel 

159 overscan regions are reversed? Default is False. 

160 

161 Returns 

162 ------- 

163 overscanResults : `lsst.pipe.base.Struct` 

164 Result struct with components: 

165 

166 ``imageFit`` 

167 Value or fit subtracted from the amplifier image data 

168 (scalar or `lsst.afw.image.Image`). 

169 ``overscanFit`` 

170 Value or fit subtracted from the serial overscan image 

171 data (scalar or `lsst.afw.image.Image`). 

172 ``overscanImage`` 

173 Image of the serial overscan region with the serial 

174 overscan correction applied 

175 (`lsst.afw.image.Image`). This quantity is used to 

176 estimate the amplifier read noise empirically. 

177 ``parallelOverscanFit`` 

178 Value or fit subtracted from the parallel overscan 

179 image data (scalar, `lsst.afw.image.Image`, or None). 

180 ``parallelOverscanImage`` 

181 Image of the parallel overscan region with the 

182 parallel overscan correction applied 

183 (`lsst.afw.image.Image` or None). 

184 

185 Raises 

186 ------ 

187 RuntimeError 

188 Raised if an invalid overscan type is set. 

189 """ 

190 # Do Serial overscan first. 

191 serialOverscanBBox = amp.getRawSerialOverscanBBox() 

192 imageBBox = amp.getRawDataBBox() 

193 

194 if self.config.doParallelOverscan: 

195 # We need to extend the serial overscan BBox to the full 

196 # size of the detector. 

197 parallelOverscanBBox = amp.getRawParallelOverscanBBox() 

198 imageBBox = imageBBox.expandedTo(parallelOverscanBBox) 

199 

200 serialOverscanBBox = geom.Box2I(geom.Point2I(serialOverscanBBox.getMinX(), 

201 imageBBox.getMinY()), 

202 geom.Extent2I(serialOverscanBBox.getWidth(), 

203 imageBBox.getHeight())) 

204 serialResults = self.correctOverscan(exposure, amp, 

205 imageBBox, serialOverscanBBox, isTransposed=isTransposed) 

206 overscanMean = serialResults.overscanMean 

207 overscanMedian = serialResults.overscanMedian 

208 overscanSigma = serialResults.overscanSigma 

209 residualMean = serialResults.overscanMeanResidual 

210 residualMedian = serialResults.overscanMedianResidual 

211 residualSigma = serialResults.overscanSigmaResidual 

212 

213 # Do Parallel Overscan 

214 parallelResults = None 

215 if self.config.doParallelOverscan: 

216 # This does not need any extensions, as we'll only 

217 # subtract it from the data region. 

218 parallelOverscanBBox = amp.getRawParallelOverscanBBox() 

219 imageBBox = amp.getRawDataBBox() 

220 

221 maskIm = exposure.getMaskedImage() 

222 maskIm = maskIm.Factory(maskIm, parallelOverscanBBox) 

223 

224 # The serial overscan correction has removed the majority 

225 # of the signal in the parallel overscan region, so the 

226 # mean should be close to zero. The noise in both should 

227 # be similar, so we can use the noise from the serial 

228 # overscan region to set the threshold for bleed 

229 # detection. 

230 thresholdLevel = self.config.numSigmaClip * serialResults.overscanSigmaResidual 

231 makeThresholdMask(maskIm, threshold=thresholdLevel, growFootprints=0) 

232 maskPix = countMaskedPixels(maskIm, self.config.maskPlanes) 

233 xSize, ySize = parallelOverscanBBox.getDimensions() 

234 if maskPix > xSize*ySize*self.config.parallelOverscanMaskThreshold: 

235 self.log.warning('Fraction of masked pixels for parallel overscan calculation larger' 

236 ' than %f of total pixels (i.e. %f masked pixels) on amp %s.', 

237 self.config.parallelOverscanMaskThreshold, maskPix, amp.getName()) 

238 self.log.warning('Not doing parallel overscan correction.') 

239 else: 

240 parallelResults = self.correctOverscan(exposure, amp, 

241 imageBBox, parallelOverscanBBox, 

242 isTransposed=not isTransposed) 

243 

244 overscanMean = (overscanMean, parallelResults.overscanMean) 

245 overscanMedian = (overscanMedian, parallelResults.overscanMedian) 

246 overscanSigma = (overscanSigma, parallelResults.overscanSigma) 

247 residualMean = (residualMean, parallelResults.overscanMeanResidual) 

248 residualMedian = (residualMedian, parallelResults.overscanMedianResidual) 

249 residualSigma = (residualSigma, parallelResults.overscanSigmaResidual) 

250 parallelOverscanFit = parallelResults.overscanOverscanModel if parallelResults else None 

251 parallelOverscanImage = parallelResults.overscanImage if parallelResults else None 

252 

253 return pipeBase.Struct(imageFit=serialResults.ampOverscanModel, 

254 overscanFit=serialResults.overscanOverscanModel, 

255 overscanImage=serialResults.overscanImage, 

256 

257 parallelOverscanFit=parallelOverscanFit, 

258 parallelOverscanImage=parallelOverscanImage, 

259 overscanMean=overscanMean, 

260 overscanMedian=overscanMedian, 

261 overscanSigma=overscanSigma, 

262 residualMean=residualMean, 

263 residualMedian=residualMedian, 

264 residualSigma=residualSigma) 

265 

266 def correctOverscan(self, exposure, amp, imageBBox, overscanBBox, isTransposed=True): 

267 """ 

268 """ 

269 overscanBox = self.trimOverscan(exposure, amp, overscanBBox, 

270 self.config.leadingColumnsToSkip, 

271 self.config.trailingColumnsToSkip, 

272 transpose=isTransposed) 

273 overscanImage = exposure[overscanBox].getMaskedImage() 

274 overscanArray = overscanImage.image.array 

275 

276 # Mask pixels. 

277 maskVal = overscanImage.mask.getPlaneBitMask(self.config.maskPlanes) 

278 overscanMask = ~((overscanImage.mask.array & maskVal) == 0) 

279 

280 median = np.ma.median(np.ma.masked_where(overscanMask, overscanArray)) 

281 bad = np.where(np.abs(overscanArray - median) > self.config.maxDeviation) 

282 overscanMask[bad] = overscanImage.mask.getPlaneBitMask("SAT") 

283 

284 # Do overscan fit. 

285 # CZW: Handle transposed correctly. 

286 overscanResults = self.fitOverscan(overscanImage, isTransposed=isTransposed) 

287 

288 # Correct image region (and possibly parallel-overscan region). 

289 ampImage = exposure[imageBBox] 

290 ampOverscanModel = self.broadcastFitToImage(overscanResults.overscanValue, 

291 ampImage.image.array, 

292 transpose=isTransposed) 

293 ampImage.image.array -= ampOverscanModel 

294 

295 # Correct overscan region (and possibly doubly-overscaned 

296 # region). 

297 overscanImage = exposure[overscanBBox] 

298 # CZW: Transposed? 

299 overscanOverscanModel = self.broadcastFitToImage(overscanResults.overscanValue, 

300 overscanImage.image.array) 

301 overscanImage.image.array -= overscanOverscanModel 

302 

303 self.debugView(overscanImage, overscanResults.overscanValue, amp) 

304 

305 # Find residual fit statistics. 

306 stats = afwMath.makeStatistics(overscanImage.getMaskedImage(), 

307 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP, self.statControl) 

308 residualMean = stats.getValue(afwMath.MEAN) 

309 residualMedian = stats.getValue(afwMath.MEDIAN) 

310 residualSigma = stats.getValue(afwMath.STDEVCLIP) 

311 

312 return pipeBase.Struct(ampOverscanModel=ampOverscanModel, 

313 overscanOverscanModel=overscanOverscanModel, 

314 overscanImage=overscanImage, 

315 overscanValue=overscanResults.overscanValue, 

316 

317 overscanMean=overscanResults.overscanMean, 

318 overscanMedian=overscanResults.overscanMedian, 

319 overscanSigma=overscanResults.overscanSigma, 

320 overscanMeanResidual=residualMean, 

321 overscanMedianResidual=residualMedian, 

322 overscanSigmaResidual=residualSigma 

323 ) 

324 

325 def broadcastFitToImage(self, overscanValue, imageArray, transpose=False): 

326 """Broadcast 0 or 1 dimension fit to appropriate shape. 

327 

328 Parameters 

329 ---------- 

330 overscanValue : `numpy.ndarray`, (Nrows, ) or scalar 

331 Overscan fit to broadcast. 

332 imageArray : `numpy.ndarray`, (Nrows, Ncols) 

333 Image array that we want to match. 

334 transpose : `bool`, optional 

335 Switch order to broadcast along the other axis. 

336 

337 Returns 

338 ------- 

339 overscanModel : `numpy.ndarray`, (Nrows, Ncols) or scalar 

340 Expanded overscan fit. 

341 

342 Raises 

343 ------ 

344 RuntimeError 

345 Raised if no axis has the appropriate dimension. 

346 """ 

347 if isinstance(overscanValue, np.ndarray): 

348 overscanModel = np.zeros_like(imageArray) 

349 

350 if transpose is False: 

351 if imageArray.shape[0] == overscanValue.shape[0]: 

352 overscanModel[:, :] = overscanValue[:, np.newaxis] 

353 elif imageArray.shape[1] == overscanValue.shape[0]: 

354 overscanModel[:, :] = overscanValue[np.newaxis, :] 

355 elif imageArray.shape[0] == overscanValue.shape[1]: 

356 overscanModel[:, :] = overscanValue[np.newaxis, :] 

357 else: 

358 raise RuntimeError(f"Could not broadcast {overscanValue.shape} to " 

359 f"match {imageArray.shape}") 

360 else: 

361 if imageArray.shape[1] == overscanValue.shape[0]: 

362 overscanModel[:, :] = overscanValue[np.newaxis, :] 

363 elif imageArray.shape[0] == overscanValue.shape[0]: 

364 overscanModel[:, :] = overscanValue[:, np.newaxis] 

365 elif imageArray.shape[1] == overscanValue.shape[1]: 

366 overscanModel[:, :] = overscanValue[:, np.newaxis] 

367 else: 

368 raise RuntimeError(f"Could not broadcast {overscanValue.shape} to " 

369 f"match {imageArray.shape}") 

370 else: 

371 overscanModel = overscanValue 

372 

373 return overscanModel 

374 

375 def trimOverscan(self, exposure, amp, bbox, skipLeading, skipTrailing, transpose=False): 

376 """Trim overscan region to remove edges. 

377 

378 Parameters 

379 ---------- 

380 exposure : `lsst.afw.image.Exposure` 

381 Exposure containing data. 

382 amp : `lsst.afw.cameraGeom.Amplifier` 

383 Amplifier containing geometry information. 

384 bbox : `lsst.geom.Box2I` 

385 Bounding box of the overscan region. 

386 skipLeading : `int` 

387 Number of leading (towards data region) rows/columns to skip. 

388 skipTrailing : `int` 

389 Number of trailing (away from data region) rows/columns to skip. 

390 transpose : `bool`, optional 

391 Operate on the transposed array. 

392 

393 Returns 

394 ------- 

395 overscanArray : `numpy.array`, (N, M) 

396 Data array to fit. 

397 overscanMask : `numpy.array`, (N, M) 

398 Data mask. 

399 """ 

400 dx0, dy0, dx1, dy1 = (0, 0, 0, 0) 

401 dataBBox = amp.getRawDataBBox() 

402 if transpose: 

403 if dataBBox.getBeginY() < bbox.getBeginY(): 

404 dy0 += skipLeading 

405 dy1 -= skipTrailing 

406 else: 

407 dy0 += skipTrailing 

408 dy1 -= skipLeading 

409 else: 

410 if dataBBox.getBeginX() < bbox.getBeginX(): 

411 dx0 += skipLeading 

412 dx1 -= skipTrailing 

413 else: 

414 dx0 += skipTrailing 

415 dx1 -= skipLeading 

416 

417 overscanBBox = geom.Box2I(bbox.getBegin() + geom.Extent2I(dx0, dy0), 

418 geom.Extent2I(bbox.getWidth() - dx0 + dx1, 

419 bbox.getHeight() - dy0 + dy1)) 

420 return overscanBBox 

421 

422 def fitOverscan(self, overscanImage, isTransposed=False): 

423 if self.config.fitType in ('MEAN', 'MEANCLIP', 'MEDIAN'): 

424 # Transposition has no effect here. 

425 overscanResult = self.measureConstantOverscan(overscanImage) 

426 overscanValue = overscanResult.overscanValue 

427 overscanMean = overscanValue 

428 overscanMedian = overscanValue 

429 overscanSigma = 0.0 

430 elif self.config.fitType in ('MEDIAN_PER_ROW', 'POLY', 'CHEB', 'LEG', 

431 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'): 

432 # Force transposes as needed 

433 overscanResult = self.measureVectorOverscan(overscanImage, isTransposed) 

434 overscanValue = overscanResult.overscanValue 

435 

436 stats = afwMath.makeStatistics(overscanResult.overscanValue, 

437 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP, 

438 self.statControl) 

439 overscanMean = stats.getValue(afwMath.MEAN) 

440 overscanMedian = stats.getValue(afwMath.MEDIAN) 

441 overscanSigma = stats.getValue(afwMath.STDEVCLIP) 

442 else: 

443 raise ValueError('%s : %s an invalid overscan type' % 

444 ("overscanCorrection", self.config.fitType)) 

445 

446 return pipeBase.Struct(overscanValue=overscanValue, 

447 overscanMean=overscanMean, 

448 overscanMedian=overscanMedian, 

449 overscanSigma=overscanSigma, 

450 ) 

451 

452 @staticmethod 

453 def integerConvert(image): 

454 """Return an integer version of the input image. 

455 

456 Parameters 

457 ---------- 

458 image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

459 Image to convert to integers. 

460 

461 Returns 

462 ------- 

463 outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage` 

464 The integer converted image. 

465 

466 Raises 

467 ------ 

468 RuntimeError 

469 Raised if the input image could not be converted. 

470 """ 

471 if hasattr(image, "image"): 

472 # Is a maskedImage: 

473 imageI = image.image.convertI() 

474 outI = afwImage.MaskedImageI(imageI, image.mask, image.variance) 

475 elif hasattr(image, "convertI"): 

476 # Is an Image: 

477 outI = image.convertI() 

478 elif hasattr(image, "astype"): 

479 # Is a numpy array: 

480 outI = image.astype(int) 

481 else: 

482 raise RuntimeError("Could not convert this to integers: %s %s %s", 

483 image, type(image), dir(image)) 

484 return outI 

485 

486 # Constant methods 

487 def measureConstantOverscan(self, image): 

488 """Measure a constant overscan value. 

489 

490 Parameters 

491 ---------- 

492 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

493 Image data to measure the overscan from. 

494 

495 Returns 

496 ------- 

497 results : `lsst.pipe.base.Struct` 

498 Overscan result with entries: 

499 - ``overscanValue``: Overscan value to subtract (`float`) 

500 - ``isTransposed``: Orientation of the overscan (`bool`) 

501 """ 

502 if self.config.fitType == 'MEDIAN': 

503 calcImage = self.integerConvert(image) 

504 else: 

505 calcImage = image 

506 fitType = afwMath.stringToStatisticsProperty(self.config.fitType) 

507 overscanValue = afwMath.makeStatistics(calcImage, fitType, self.statControl).getValue() 

508 

509 return pipeBase.Struct(overscanValue=overscanValue, 

510 isTransposed=False) 

511 

512 # Vector correction utilities 

513 def getImageArray(self, image): 

514 """Extract the numpy array from the input image. 

515 

516 Parameters 

517 ---------- 

518 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

519 Image data to pull array from. 

520 

521 calcImage : `numpy.ndarray` 

522 Image data array for numpy operating. 

523 """ 

524 if hasattr(image, "getImage"): 

525 calcImage = image.getImage().getArray() 

526 calcImage = np.ma.masked_where(image.getMask().getArray() & self.statControl.getAndMask(), 

527 calcImage) 

528 else: 

529 calcImage = image.getArray() 

530 return calcImage 

531 

532 def maskOutliers(self, imageArray): 

533 """Mask outliers in a row of overscan data from a robust sigma 

534 clipping procedure. 

535 

536 Parameters 

537 ---------- 

538 imageArray : `numpy.ndarray` 

539 Image to filter along numpy axis=1. 

540 

541 Returns 

542 ------- 

543 maskedArray : `numpy.ma.masked_array` 

544 Masked image marking outliers. 

545 """ 

546 lq, median, uq = np.percentile(imageArray, [25.0, 50.0, 75.0], axis=1) 

547 axisMedians = median 

548 axisStdev = 0.74*(uq - lq) # robust stdev 

549 

550 diff = np.abs(imageArray - axisMedians[:, np.newaxis]) 

551 return np.ma.masked_where(diff > self.statControl.getNumSigmaClip() 

552 * axisStdev[:, np.newaxis], imageArray) 

553 

554 @staticmethod 

555 def collapseArray(maskedArray): 

556 """Collapse overscan array (and mask) to a 1-D vector of values. 

557 

558 Parameters 

559 ---------- 

560 maskedArray : `numpy.ma.masked_array` 

561 Masked array of input overscan data. 

562 

563 Returns 

564 ------- 

565 collapsed : `numpy.ma.masked_array` 

566 Single dimensional overscan data, combined with the mean. 

567 """ 

568 collapsed = np.mean(maskedArray, axis=1) 

569 if collapsed.mask.sum() > 0: 

570 collapsed.data[collapsed.mask] = np.mean(maskedArray.data[collapsed.mask], axis=1) 

571 return collapsed 

572 

573 def collapseArrayMedian(self, maskedArray): 

574 """Collapse overscan array (and mask) to a 1-D vector of using the 

575 correct integer median of row-values. 

576 

577 Parameters 

578 ---------- 

579 maskedArray : `numpy.ma.masked_array` 

580 Masked array of input overscan data. 

581 

582 Returns 

583 ------- 

584 collapsed : `numpy.ma.masked_array` 

585 Single dimensional overscan data, combined with the afwMath median. 

586 """ 

587 integerMI = self.integerConvert(maskedArray) 

588 

589 collapsed = [] 

590 fitType = afwMath.stringToStatisticsProperty('MEDIAN') 

591 for row in integerMI: 

592 newRow = row.compressed() 

593 if len(newRow) > 0: 

594 rowMedian = afwMath.makeStatistics(newRow, fitType, self.statControl).getValue() 

595 else: 

596 rowMedian = np.nan 

597 collapsed.append(rowMedian) 

598 

599 return np.array(collapsed) 

600 

601 def splineFit(self, indices, collapsed, numBins): 

602 """Wrapper function to match spline fit API to polynomial fit API. 

603 

604 Parameters 

605 ---------- 

606 indices : `numpy.ndarray` 

607 Locations to evaluate the spline. 

608 collapsed : `numpy.ndarray` 

609 Collapsed overscan values corresponding to the spline 

610 evaluation points. 

611 numBins : `int` 

612 Number of bins to use in constructing the spline. 

613 

614 Returns 

615 ------- 

616 interp : `lsst.afw.math.Interpolate` 

617 Interpolation object for later evaluation. 

618 """ 

619 if not np.ma.is_masked(collapsed): 

620 collapsed.mask = np.array(len(collapsed)*[np.ma.nomask]) 

621 

622 numPerBin, binEdges = np.histogram(indices, bins=numBins, 

623 weights=1 - collapsed.mask.astype(int)) 

624 with np.errstate(invalid="ignore"): 

625 values = np.histogram(indices, bins=numBins, 

626 weights=collapsed.data*~collapsed.mask)[0]/numPerBin 

627 binCenters = np.histogram(indices, bins=numBins, 

628 weights=indices*~collapsed.mask)[0]/numPerBin 

629 

630 if len(binCenters[numPerBin > 0]) < 5: 

631 self.log.warn("Cannot do spline fitting for overscan: %s valid points.", 

632 len(binCenters[numPerBin > 0])) 

633 # Return a scalar value if we have one, otherwise 

634 # return zero. This amplifier is hopefully already 

635 # masked. 

636 if len(values[numPerBin > 0]) != 0: 

637 return float(values[numPerBin > 0][0]) 

638 else: 

639 return 0.0 

640 

641 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0], 

642 values.astype(float)[numPerBin > 0], 

643 afwMath.stringToInterpStyle(self.config.fitType)) 

644 return interp 

645 

646 @staticmethod 

647 def splineEval(indices, interp): 

648 """Wrapper function to match spline evaluation API to polynomial fit 

649 API. 

650 

651 Parameters 

652 ---------- 

653 indices : `numpy.ndarray` 

654 Locations to evaluate the spline. 

655 interp : `lsst.afw.math.interpolate` 

656 Interpolation object to use. 

657 

658 Returns 

659 ------- 

660 values : `numpy.ndarray` 

661 Evaluated spline values at each index. 

662 """ 

663 

664 return interp.interpolate(indices.astype(float)) 

665 

666 @staticmethod 

667 def maskExtrapolated(collapsed): 

668 """Create mask if edges are extrapolated. 

669 

670 Parameters 

671 ---------- 

672 collapsed : `numpy.ma.masked_array` 

673 Masked array to check the edges of. 

674 

675 Returns 

676 ------- 

677 maskArray : `numpy.ndarray` 

678 Boolean numpy array of pixels to mask. 

679 """ 

680 maskArray = np.full_like(collapsed, False, dtype=bool) 

681 if np.ma.is_masked(collapsed): 

682 num = len(collapsed) 

683 for low in range(num): 

684 if not collapsed.mask[low]: 

685 break 

686 if low > 0: 

687 maskArray[:low] = True 

688 for high in range(1, num): 

689 if not collapsed.mask[-high]: 

690 break 

691 if high > 1: 

692 maskArray[-high:] = True 

693 return maskArray 

694 

695 def measureVectorOverscan(self, image, isTransposed=False): 

696 """Calculate the 1-d vector overscan from the input overscan image. 

697 

698 Parameters 

699 ---------- 

700 image : `lsst.afw.image.MaskedImage` 

701 Image containing the overscan data. 

702 isTransposed : `bool` 

703 If true, the image has been transposed. 

704 

705 Returns 

706 ------- 

707 results : `lsst.pipe.base.Struct` 

708 Overscan result with entries: 

709 

710 ``overscanValue`` 

711 Overscan value to subtract (`float`) 

712 ``maskArray`` 

713 List of rows that should be masked as ``SUSPECT`` when the 

714 overscan solution is applied. (`list` [ `bool` ]) 

715 ``isTransposed`` 

716 Indicates if the overscan data was transposed during 

717 calcuation, noting along which axis the overscan should be 

718 subtracted. (`bool`) 

719 """ 

720 calcImage = self.getImageArray(image) 

721 

722 # operate on numpy-arrays from here 

723 if isTransposed: 

724 calcImage = np.transpose(calcImage) 

725 masked = self.maskOutliers(calcImage) 

726 

727 if self.config.fitType == 'MEDIAN_PER_ROW': 

728 mi = afwImage.MaskedImageI(image.getBBox()) 

729 masked = masked.astype(int) 

730 if isTransposed: 

731 masked = masked.transpose() 

732 

733 mi.image.array[:, :] = masked.data[:, :] 

734 if bool(masked.mask.shape): 

735 mi.mask.array[:, :] = masked.mask[:, :] 

736 

737 overscanVector = fitOverscanImage(mi, self.config.maskPlanes, isTransposed) 

738 maskArray = self.maskExtrapolated(overscanVector) 

739 else: 

740 collapsed = self.collapseArray(masked) 

741 

742 num = len(collapsed) 

743 indices = 2.0*np.arange(num)/float(num) - 1.0 

744 

745 poly = np.polynomial 

746 fitter, evaler = { 

747 'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval), 

748 'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval), 

749 'LEG': (poly.legendre.legfit, poly.legendre.legval), 

750 'NATURAL_SPLINE': (self.splineFit, self.splineEval), 

751 'CUBIC_SPLINE': (self.splineFit, self.splineEval), 

752 'AKIMA_SPLINE': (self.splineFit, self.splineEval) 

753 }[self.config.fitType] 

754 

755 # These are the polynomial coefficients, or an 

756 # interpolation object. 

757 coeffs = fitter(indices, collapsed, self.config.order) 

758 

759 if isinstance(coeffs, float): 

760 self.log.warn("Using fallback value %f due to fitter failure. Amplifier will be masked.", 

761 coeffs) 

762 overscanVector = np.full_like(indices, coeffs) 

763 maskArray = np.full_like(collapsed, True, dtype=bool) 

764 else: 

765 # Otherwise we can just use things as normal. 

766 overscanVector = evaler(indices, coeffs) 

767 maskArray = self.maskExtrapolated(collapsed) 

768 

769 return pipeBase.Struct(overscanValue=np.array(overscanVector), 

770 maskArray=maskArray, 

771 isTransposed=isTransposed) 

772 

773 def debugView(self, image, model, amp=None): 

774 """Debug display for the final overscan solution. 

775 

776 Parameters 

777 ---------- 

778 image : `lsst.afw.image.Image` 

779 Input image the overscan solution was determined from. 

780 model : `numpy.ndarray` or `float` 

781 Overscan model determined for the image. 

782 amp : `lsst.afw.cameraGeom.Amplifier`, optional 

783 Amplifier to extract diagnostic information. 

784 """ 

785 import lsstDebug 

786 if not lsstDebug.Info(__name__).display: 

787 return 

788 if not self.allowDebug: 

789 return 

790 

791 calcImage = self.getImageArray(image) 

792 # CZW: Check that this is ok 

793 calcImage = np.transpose(calcImage) 

794 masked = self.maskOutliers(calcImage) 

795 collapsed = self.collapseArray(masked) 

796 

797 num = len(collapsed) 

798 indices = 2.0 * np.arange(num)/float(num) - 1.0 

799 

800 if np.ma.is_masked(collapsed): 

801 collapsedMask = collapsed.mask 

802 else: 

803 collapsedMask = np.array(num*[np.ma.nomask]) 

804 

805 import matplotlib.pyplot as plot 

806 figure = plot.figure(1) 

807 figure.clear() 

808 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

809 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+') 

810 if collapsedMask.sum() > 0: 

811 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+') 

812 if isinstance(model, np.ndarray): 

813 plotModel = model 

814 else: 

815 plotModel = np.zeros_like(indices) 

816 plotModel += model 

817 axes.plot(indices, plotModel, 'r-') 

818 plot.xlabel("centered/scaled position along overscan region") 

819 plot.ylabel("pixel value/fit value") 

820 if amp: 

821 plot.title(f"{amp.getName()} DataX: " 

822 f"[{amp.getRawDataBBox().getBeginX()}:{amp.getRawBBox().getEndX()}]" 

823 f"OscanX: [{amp.getRawHorizontalOverscanBBox().getBeginX()}:" 

824 f"{amp.getRawHorizontalOverscanBBox().getEndX()}] {self.config.fitType}") 

825 else: 

826 plot.title("No amp supplied.") 

827 figure.show() 

828 prompt = "Press Enter or c to continue [chp]..." 

829 while True: 

830 ans = input(prompt).lower() 

831 if ans in ("", " ", "c",): 

832 break 

833 elif ans in ("p", ): 

834 import pdb 

835 pdb.set_trace() 

836 elif ans in ('x', ): 

837 self.allowDebug = False 

838 break 

839 elif ans in ("h", ): 

840 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

841 plot.close()