Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23import numpy 

24from deprecated.sphinx import deprecated 

25 

26import lsst.geom 

27import lsst.afw.image as afwImage 

28import lsst.afw.detection as afwDetection 

29import lsst.afw.math as afwMath 

30import lsst.meas.algorithms as measAlg 

31import lsst.pex.exceptions as pexExcept 

32import lsst.afw.cameraGeom as camGeom 

33 

34from lsst.afw.geom.wcsUtils import makeDistortedTanWcs 

35from lsst.meas.algorithms.detection import SourceDetectionTask 

36from lsst.pipe.base import Struct 

37 

38from contextlib import contextmanager 

39 

40 

41def createPsf(fwhm): 

42 """Make a double Gaussian PSF. 

43 

44 Parameters 

45 ---------- 

46 fwhm : scalar 

47 FWHM of double Gaussian smoothing kernel. 

48 

49 Returns 

50 ------- 

51 psf : `lsst.meas.algorithms.DoubleGaussianPsf` 

52 The created smoothing kernel. 

53 """ 

54 ksize = 4*int(fwhm) + 1 

55 return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2)))) 

56 

57 

58def transposeMaskedImage(maskedImage): 

59 """Make a transposed copy of a masked image. 

60 

61 Parameters 

62 ---------- 

63 maskedImage : `lsst.afw.image.MaskedImage` 

64 Image to process. 

65 

66 Returns 

67 ------- 

68 transposed : `lsst.afw.image.MaskedImage` 

69 The transposed copy of the input image. 

70 """ 

71 transposed = maskedImage.Factory(lsst.geom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth())) 

72 transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T 

73 transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T 

74 transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T 

75 return transposed 

76 

77 

78def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None): 

79 """Interpolate over defects specified in a defect list. 

80 

81 Parameters 

82 ---------- 

83 maskedImage : `lsst.afw.image.MaskedImage` 

84 Image to process. 

85 defectList : `lsst.meas.algorithms.Defects` 

86 List of defects to interpolate over. 

87 fwhm : scalar 

88 FWHM of double Gaussian smoothing kernel. 

89 fallbackValue : scalar, optional 

90 Fallback value if an interpolated value cannot be determined. 

91 If None, then the clipped mean of the image is used. 

92 """ 

93 psf = createPsf(fwhm) 

94 if fallbackValue is None: 

95 fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue() 

96 if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict(): 

97 maskedImage.getMask().addMaskPlane('INTRP') 

98 measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True) 

99 return maskedImage 

100 

101 

102def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'): 

103 """Mask pixels based on threshold detection. 

104 

105 Parameters 

106 ---------- 

107 maskedImage : `lsst.afw.image.MaskedImage` 

108 Image to process. Only the mask plane is updated. 

109 threshold : scalar 

110 Detection threshold. 

111 growFootprints : scalar, optional 

112 Number of pixels to grow footprints of detected regions. 

113 maskName : str, optional 

114 Mask plane name, or list of names to convert 

115 

116 Returns 

117 ------- 

118 defectList : `lsst.meas.algorithms.Defects` 

119 Defect list constructed from pixels above the threshold. 

120 """ 

121 # find saturated regions 

122 thresh = afwDetection.Threshold(threshold) 

123 fs = afwDetection.FootprintSet(maskedImage, thresh) 

124 

125 if growFootprints > 0: 

126 fs = afwDetection.FootprintSet(fs, rGrow=growFootprints, isotropic=False) 

127 fpList = fs.getFootprints() 

128 

129 # set mask 

130 mask = maskedImage.getMask() 

131 bitmask = mask.getPlaneBitMask(maskName) 

132 afwDetection.setMaskFromFootprintList(mask, fpList, bitmask) 

133 

134 return measAlg.Defects.fromFootprintList(fpList) 

135 

136 

137def growMasks(mask, radius=0, maskNameList=['BAD'], maskValue="BAD"): 

138 """Grow a mask by an amount and add to the requested plane. 

139 

140 Parameters 

141 ---------- 

142 mask : `lsst.afw.image.Mask` 

143 Mask image to process. 

144 radius : scalar 

145 Amount to grow the mask. 

146 maskNameList : `str` or `list` [`str`] 

147 Mask names that should be grown. 

148 maskValue : `str` 

149 Mask plane to assign the newly masked pixels to. 

150 """ 

151 if radius > 0: 

152 thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK) 

153 fpSet = afwDetection.FootprintSet(mask, thresh) 

154 fpSet = afwDetection.FootprintSet(fpSet, rGrow=radius, isotropic=False) 

155 fpSet.setMask(mask, maskValue) 

156 

157 

158def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1, 

159 maskNameList=['SAT'], fallbackValue=None): 

160 """Interpolate over defects identified by a particular set of mask planes. 

161 

162 Parameters 

163 ---------- 

164 maskedImage : `lsst.afw.image.MaskedImage` 

165 Image to process. 

166 fwhm : scalar 

167 FWHM of double Gaussian smoothing kernel. 

168 growSaturatedFootprints : scalar, optional 

169 Number of pixels to grow footprints for saturated pixels. 

170 maskNameList : `List` of `str`, optional 

171 Mask plane name. 

172 fallbackValue : scalar, optional 

173 Value of last resort for interpolation. 

174 """ 

175 mask = maskedImage.getMask() 

176 

177 if growSaturatedFootprints > 0 and "SAT" in maskNameList: 

178 # If we are interpolating over an area larger than the original masked region, we need 

179 # to expand the original mask bit to the full area to explain why we interpolated there. 

180 growMasks(mask, radius=growSaturatedFootprints, maskNameList=['SAT'], maskValue="SAT") 

181 

182 thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK) 

183 fpSet = afwDetection.FootprintSet(mask, thresh) 

184 defectList = measAlg.Defects.fromFootprintList(fpSet.getFootprints()) 

185 

186 interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue) 

187 

188 return maskedImage 

189 

190 

191def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', 

192 fallbackValue=None): 

193 """Mark saturated pixels and optionally interpolate over them 

194 

195 Parameters 

196 ---------- 

197 maskedImage : `lsst.afw.image.MaskedImage` 

198 Image to process. 

199 saturation : scalar 

200 Saturation level used as the detection threshold. 

201 fwhm : scalar 

202 FWHM of double Gaussian smoothing kernel. 

203 growFootprints : scalar, optional 

204 Number of pixels to grow footprints of detected regions. 

205 interpolate : Bool, optional 

206 If True, saturated pixels are interpolated over. 

207 maskName : str, optional 

208 Mask plane name. 

209 fallbackValue : scalar, optional 

210 Value of last resort for interpolation. 

211 """ 

212 defectList = makeThresholdMask( 

213 maskedImage=maskedImage, 

214 threshold=saturation, 

215 growFootprints=growFootprints, 

216 maskName=maskName, 

217 ) 

218 if interpolate: 

219 interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue) 

220 

221 return maskedImage 

222 

223 

224def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage): 

225 """Compute number of edge trim pixels to match the calibration data. 

226 

227 Use the dimension difference between the raw exposure and the 

228 calibration exposure to compute the edge trim pixels. This trim 

229 is applied symmetrically, with the same number of pixels masked on 

230 each side. 

231 

232 Parameters 

233 ---------- 

234 rawMaskedImage : `lsst.afw.image.MaskedImage` 

235 Image to trim. 

236 calibMaskedImage : `lsst.afw.image.MaskedImage` 

237 Calibration image to draw new bounding box from. 

238 

239 Returns 

240 ------- 

241 replacementMaskedImage : `lsst.afw.image.MaskedImage` 

242 ``rawMaskedImage`` trimmed to the appropriate size 

243 Raises 

244 ------ 

245 RuntimeError 

246 Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to 

247 match ``calibMaskedImage``. 

248 """ 

249 nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions() 

250 if nx != ny: 

251 raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.") 

252 if nx % 2 != 0: 

253 raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.") 

254 if nx < 0: 

255 raise RuntimeError("Calibration maskedImage is larger than raw data.") 

256 

257 nEdge = nx//2 

258 if nEdge > 0: 

259 replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL] 

260 SourceDetectionTask.setEdgeBits( 

261 rawMaskedImage, 

262 replacementMaskedImage.getBBox(), 

263 rawMaskedImage.getMask().getPlaneBitMask("EDGE") 

264 ) 

265 else: 

266 replacementMaskedImage = rawMaskedImage 

267 

268 return replacementMaskedImage 

269 

270 

271def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False): 

272 """Apply bias correction in place. 

273 

274 Parameters 

275 ---------- 

276 maskedImage : `lsst.afw.image.MaskedImage` 

277 Image to process. The image is modified by this method. 

278 biasMaskedImage : `lsst.afw.image.MaskedImage` 

279 Bias image of the same size as ``maskedImage`` 

280 trimToFit : `Bool`, optional 

281 If True, raw data is symmetrically trimmed to match 

282 calibration size. 

283 

284 Raises 

285 ------ 

286 RuntimeError 

287 Raised if ``maskedImage`` and ``biasMaskedImage`` do not have 

288 the same size. 

289 

290 """ 

291 if trimToFit: 

292 maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage) 

293 

294 if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL): 

295 raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" % 

296 (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL))) 

297 maskedImage -= biasMaskedImage 

298 

299 

300def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False): 

301 """Apply dark correction in place. 

302 

303 Parameters 

304 ---------- 

305 maskedImage : `lsst.afw.image.MaskedImage` 

306 Image to process. The image is modified by this method. 

307 darkMaskedImage : `lsst.afw.image.MaskedImage` 

308 Dark image of the same size as ``maskedImage``. 

309 expScale : scalar 

310 Dark exposure time for ``maskedImage``. 

311 darkScale : scalar 

312 Dark exposure time for ``darkMaskedImage``. 

313 invert : `Bool`, optional 

314 If True, re-add the dark to an already corrected image. 

315 trimToFit : `Bool`, optional 

316 If True, raw data is symmetrically trimmed to match 

317 calibration size. 

318 

319 Raises 

320 ------ 

321 RuntimeError 

322 Raised if ``maskedImage`` and ``darkMaskedImage`` do not have 

323 the same size. 

324 

325 Notes 

326 ----- 

327 The dark correction is applied by calculating: 

328 maskedImage -= dark * expScaling / darkScaling 

329 """ 

330 if trimToFit: 

331 maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage) 

332 

333 if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL): 

334 raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" % 

335 (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL))) 

336 

337 scale = expScale / darkScale 

338 if not invert: 

339 maskedImage.scaledMinus(scale, darkMaskedImage) 

340 else: 

341 maskedImage.scaledPlus(scale, darkMaskedImage) 

342 

343 

344def updateVariance(maskedImage, gain, readNoise): 

345 """Set the variance plane based on the image plane. 

346 

347 Parameters 

348 ---------- 

349 maskedImage : `lsst.afw.image.MaskedImage` 

350 Image to process. The variance plane is modified. 

351 gain : scalar 

352 The amplifier gain in electrons/ADU. 

353 readNoise : scalar 

354 The amplifier read nmoise in ADU/pixel. 

355 """ 

356 var = maskedImage.getVariance() 

357 var[:] = maskedImage.getImage() 

358 var /= gain 

359 var += readNoise**2 

360 

361 

362def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False): 

363 """Apply flat correction in place. 

364 

365 Parameters 

366 ---------- 

367 maskedImage : `lsst.afw.image.MaskedImage` 

368 Image to process. The image is modified. 

369 flatMaskedImage : `lsst.afw.image.MaskedImage` 

370 Flat image of the same size as ``maskedImage`` 

371 scalingType : str 

372 Flat scale computation method. Allowed values are 'MEAN', 

373 'MEDIAN', or 'USER'. 

374 userScale : scalar, optional 

375 Scale to use if ``scalingType``='USER'. 

376 invert : `Bool`, optional 

377 If True, unflatten an already flattened image. 

378 trimToFit : `Bool`, optional 

379 If True, raw data is symmetrically trimmed to match 

380 calibration size. 

381 

382 Raises 

383 ------ 

384 RuntimeError 

385 Raised if ``maskedImage`` and ``flatMaskedImage`` do not have 

386 the same size or if ``scalingType`` is not an allowed value. 

387 """ 

388 if trimToFit: 

389 maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage) 

390 

391 if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL): 

392 raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" % 

393 (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL))) 

394 

395 # Figure out scale from the data 

396 # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility 

397 # in the case that the flat is created by some other mechanism. 

398 if scalingType in ('MEAN', 'MEDIAN'): 

399 scalingType = afwMath.stringToStatisticsProperty(scalingType) 

400 flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue() 

401 elif scalingType == 'USER': 

402 flatScale = userScale 

403 else: 

404 raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType)) 

405 

406 if not invert: 

407 maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage) 

408 else: 

409 maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage) 

410 

411 

412def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True): 

413 """Apply illumination correction in place. 

414 

415 Parameters 

416 ---------- 

417 maskedImage : `lsst.afw.image.MaskedImage` 

418 Image to process. The image is modified. 

419 illumMaskedImage : `lsst.afw.image.MaskedImage` 

420 Illumination correction image of the same size as ``maskedImage``. 

421 illumScale : scalar 

422 Scale factor for the illumination correction. 

423 trimToFit : `Bool`, optional 

424 If True, raw data is symmetrically trimmed to match 

425 calibration size. 

426 

427 Raises 

428 ------ 

429 RuntimeError 

430 Raised if ``maskedImage`` and ``illumMaskedImage`` do not have 

431 the same size. 

432 """ 

433 if trimToFit: 

434 maskedImage = trimToMatchCalibBBox(maskedImage, illumMaskedImage) 

435 

436 if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL): 

437 raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" % 

438 (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL))) 

439 

440 maskedImage.scaledDivides(1.0/illumScale, illumMaskedImage) 

441 

442 

443def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, 

444 statControl=None, overscanIsInt=True): 

445 """Apply overscan correction in place. 

446 

447 Parameters 

448 ---------- 

449 ampMaskedImage : `lsst.afw.image.MaskedImage` 

450 Image of amplifier to correct; modified. 

451 overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

452 Image of overscan; modified. 

453 fitType : `str` 

454 Type of fit for overscan correction. May be one of: 

455 

456 - ``MEAN``: use mean of overscan. 

457 - ``MEANCLIP``: use clipped mean of overscan. 

458 - ``MEDIAN``: use median of overscan. 

459 - ``MEDIAN_PER_ROW``: use median per row of overscan. 

460 - ``POLY``: fit with ordinary polynomial. 

461 - ``CHEB``: fit with Chebyshev polynomial. 

462 - ``LEG``: fit with Legendre polynomial. 

463 - ``NATURAL_SPLINE``: fit with natural spline. 

464 - ``CUBIC_SPLINE``: fit with cubic spline. 

465 - ``AKIMA_SPLINE``: fit with Akima spline. 

466 

467 order : `int` 

468 Polynomial order or number of spline knots; ignored unless 

469 ``fitType`` indicates a polynomial or spline. 

470 statControl : `lsst.afw.math.StatisticsControl` 

471 Statistics control object. In particular, we pay attention to numSigmaClip 

472 overscanIsInt : `bool` 

473 Treat the overscan region as consisting of integers, even if it's been 

474 converted to float. E.g. handle ties properly. 

475 

476 Returns 

477 ------- 

478 result : `lsst.pipe.base.Struct` 

479 Result struct with components: 

480 

481 - ``imageFit``: Value(s) removed from image (scalar or 

482 `lsst.afw.image.Image`) 

483 - ``overscanFit``: Value(s) removed from overscan (scalar or 

484 `lsst.afw.image.Image`) 

485 - ``overscanImage``: Overscan corrected overscan region 

486 (`lsst.afw.image.Image`) 

487 Raises 

488 ------ 

489 pexExcept.Exception 

490 Raised if ``fitType`` is not an allowed value. 

491 

492 Notes 

493 ----- 

494 The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit 

495 subtracted. Note that the ``overscanImage`` should not be a subimage of 

496 the ``ampMaskedImage``, to avoid being subtracted twice. 

497 

498 Debug plots are available for the SPLINE fitTypes by setting the 

499 `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These 

500 plots show the scatter plot of the overscan data (collapsed along 

501 the perpendicular dimension) as a function of position on the CCD 

502 (normalized between +/-1). 

503 """ 

504 ampImage = ampMaskedImage.getImage() 

505 if statControl is None: 

506 statControl = afwMath.StatisticsControl() 

507 

508 numSigmaClip = statControl.getNumSigmaClip() 

509 if fitType in ('MEAN', 'MEANCLIP'): 

510 fitType = afwMath.stringToStatisticsProperty(fitType) 

511 offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue() 

512 overscanFit = offImage 

513 elif fitType in ('MEDIAN', 'MEDIAN_PER_ROW',): 

514 if overscanIsInt: 

515 # we need an image with integer pixels to handle ties properly 

516 if hasattr(overscanImage, "image"): 

517 imageI = overscanImage.image.convertI() 

518 overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance) 

519 else: 

520 overscanImageI = overscanImage.convertI() 

521 else: 

522 overscanImageI = overscanImage 

523 if fitType in ('MEDIAN',): 

524 fitTypeStats = afwMath.stringToStatisticsProperty(fitType) 

525 offImage = afwMath.makeStatistics(overscanImageI, fitTypeStats, statControl).getValue() 

526 overscanFit = offImage 

527 elif fitType in ('MEDIAN_PER_ROW',): 

528 if hasattr(overscanImageI, "getImage"): 

529 biasArray = overscanImageI.getImage().getArray() 

530 else: 

531 biasArray = overscanImageI.getArray() 

532 shortInd = numpy.argmin(biasArray.shape) 

533 if shortInd == 0: 

534 # Convert to some 'standard' representation to make things easier 

535 biasArray = numpy.transpose(biasArray) 

536 

537 fitTypeStats = afwMath.stringToStatisticsProperty('MEDIAN') 

538 collapsed = [] 

539 for row in biasArray: 

540 rowMedian = afwMath.makeStatistics(row, fitTypeStats, statControl).getValue() 

541 collapsed.append(rowMedian) 

542 collapsed = numpy.array(collapsed) 

543 offImage = ampImage.Factory(ampImage.getDimensions()) 

544 offArray = offImage.getArray() 

545 overscanFit = afwImage.ImageF(overscanImage.getDimensions()) 

546 overscanArray = overscanFit.getArray() 

547 

548 if shortInd == 1: 

549 offArray[:, :] = collapsed[:, numpy.newaxis] 

550 overscanArray[:, :] = collapsed[:, numpy.newaxis] 

551 else: 

552 offArray[:, :] = collapsed[numpy.newaxis, :] 

553 overscanArray[:, :] = collapsed[numpy.newaxis, :] 

554 

555 del collapsed, biasArray 

556 

557 if overscanIsInt: 

558 del overscanImageI 

559 elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'): 

560 if hasattr(overscanImage, "getImage"): 

561 biasArray = overscanImage.getImage().getArray() 

562 biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(), 

563 biasArray) 

564 else: 

565 biasArray = overscanImage.getArray() 

566 # Fit along the long axis, so collapse along each short row and fit the resulting array 

567 shortInd = numpy.argmin(biasArray.shape) 

568 if shortInd == 0: 

569 # Convert to some 'standard' representation to make things easier 

570 biasArray = numpy.transpose(biasArray) 

571 

572 # Do a single round of clipping to weed out CR hits and signal leaking into the overscan 

573 percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1) 

574 medianBiasArr = percentiles[1] 

575 stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev 

576 diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis]) 

577 biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray) 

578 collapsed = numpy.mean(biasMaskedArr, axis=1) 

579 if collapsed.mask.sum() > 0: 

580 collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1) 

581 

582 del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr 

583 

584 if shortInd == 0: 

585 collapsed = numpy.transpose(collapsed) 

586 

587 num = len(collapsed) 

588 indices = 2.0*numpy.arange(num)/float(num) - 1.0 

589 

590 if fitType in ('POLY', 'CHEB', 'LEG'): 

591 # A numpy polynomial 

592 poly = numpy.polynomial 

593 fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval), 

594 "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval), 

595 "LEG": (poly.legendre.legfit, poly.legendre.legval), 

596 }[fitType] 

597 

598 coeffs = fitter(indices, collapsed, order) 

599 fitBiasArr = evaler(indices, coeffs) 

600 elif 'SPLINE' in fitType: 

601 # An afw interpolation 

602 numBins = order 

603 # 

604 # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case 

605 # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask 

606 # 

607 # Issue DM-415 

608 # 

609 collapsedMask = collapsed.mask 

610 try: 

611 if collapsedMask == numpy.ma.nomask: 

612 collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask]) 

613 except ValueError: # If collapsedMask is an array the test fails [needs .all()] 

614 pass 

615 

616 numPerBin, binEdges = numpy.histogram(indices, bins=numBins, 

617 weights=1-collapsedMask.astype(int)) 

618 # Binning is just a histogram, with weights equal to the values. 

619 # Use a similar trick to get the bin centers (this deals with different numbers per bin). 

620 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

621 values = numpy.histogram(indices, bins=numBins, 

622 weights=collapsed.data*~collapsedMask)[0]/numPerBin 

623 binCenters = numpy.histogram(indices, bins=numBins, 

624 weights=indices*~collapsedMask)[0]/numPerBin 

625 interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0], 

626 values.astype(float)[numPerBin > 0], 

627 afwMath.stringToInterpStyle(fitType)) 

628 fitBiasArr = numpy.array([interp.interpolate(i) for i in indices]) 

629 

630 import lsstDebug 

631 if lsstDebug.Info(__name__).display: 

632 import matplotlib.pyplot as plot 

633 figure = plot.figure(1) 

634 figure.clear() 

635 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

636 axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+') 

637 if collapsedMask.sum() > 0: 

638 axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+') 

639 axes.plot(indices, fitBiasArr, 'r-') 

640 plot.xlabel("centered/scaled position along overscan region") 

641 plot.ylabel("pixel value/fit value") 

642 figure.show() 

643 prompt = "Press Enter or c to continue [chp]... " 

644 while True: 

645 ans = input(prompt).lower() 

646 if ans in ("", "c",): 

647 break 

648 if ans in ("p",): 

649 import pdb 

650 pdb.set_trace() 

651 elif ans in ("h", ): 

652 print("h[elp] c[ontinue] p[db]") 

653 plot.close() 

654 

655 offImage = ampImage.Factory(ampImage.getDimensions()) 

656 offArray = offImage.getArray() 

657 overscanFit = afwImage.ImageF(overscanImage.getDimensions()) 

658 overscanArray = overscanFit.getArray() 

659 if shortInd == 1: 

660 offArray[:, :] = fitBiasArr[:, numpy.newaxis] 

661 overscanArray[:, :] = fitBiasArr[:, numpy.newaxis] 

662 else: 

663 offArray[:, :] = fitBiasArr[numpy.newaxis, :] 

664 overscanArray[:, :] = fitBiasArr[numpy.newaxis, :] 

665 

666 # We don't trust any extrapolation: mask those pixels as SUSPECT 

667 # This will occur when the top and or bottom edges of the overscan 

668 # contain saturated values. The values will be extrapolated from 

669 # the surrounding pixels, but we cannot entirely trust the value of 

670 # the extrapolation, and will mark the image mask plane to flag the 

671 # image as such. 

672 mask = ampMaskedImage.getMask() 

673 maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose() 

674 suspect = mask.getPlaneBitMask("SUSPECT") 

675 try: 

676 if collapsed.mask == numpy.ma.nomask: 

677 # There is no mask, so the whole array is fine 

678 pass 

679 except ValueError: # If collapsed.mask is an array the test fails [needs .all()] 

680 for low in range(num): 

681 if not collapsed.mask[low]: 

682 break 

683 if low > 0: 

684 maskArray[:low, :] |= suspect 

685 for high in range(1, num): 

686 if not collapsed.mask[-high]: 

687 break 

688 if high > 1: 

689 maskArray[-high:, :] |= suspect 

690 

691 else: 

692 raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType)) 

693 ampImage -= offImage 

694 overscanImage -= overscanFit 

695 return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage) 

696 

697 

698def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain, gains=None): 

699 """Apply brighter fatter correction in place for the image. 

700 

701 Parameters 

702 ---------- 

703 exposure : `lsst.afw.image.Exposure` 

704 Exposure to have brighter-fatter correction applied. Modified 

705 by this method. 

706 kernel : `numpy.ndarray` 

707 Brighter-fatter kernel to apply. 

708 maxIter : scalar 

709 Number of correction iterations to run. 

710 threshold : scalar 

711 Convergence threshold in terms of the sum of absolute 

712 deviations between an iteration and the previous one. 

713 applyGain : `Bool` 

714 If True, then the exposure values are scaled by the gain prior 

715 to correction. 

716 gains : `dict` [`str`, `float`] 

717 A dictionary, keyed by amplifier name, of the gains to use. 

718 If gains is None, the nominal gains in the amplifier object are used. 

719 

720 Returns 

721 ------- 

722 diff : `float` 

723 Final difference between iterations achieved in correction. 

724 iteration : `int` 

725 Number of iterations used to calculate correction. 

726 

727 Notes 

728 ----- 

729 This correction takes a kernel that has been derived from flat 

730 field images to redistribute the charge. The gradient of the 

731 kernel is the deflection field due to the accumulated charge. 

732 

733 Given the original image I(x) and the kernel K(x) we can compute 

734 the corrected image Ic(x) using the following equation: 

735 

736 Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 

737 

738 To evaluate the derivative term we expand it as follows: 

739 

740 0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 

741 

742 Because we use the measured counts instead of the incident counts 

743 we apply the correction iteratively to reconstruct the original 

744 counts and the correction. We stop iterating when the summed 

745 difference between the current corrected image and the one from 

746 the previous iteration is below the threshold. We do not require 

747 convergence because the number of iterations is too large a 

748 computational cost. How we define the threshold still needs to be 

749 evaluated, the current default was shown to work reasonably well 

750 on a small set of images. For more information on the method see 

751 DocuShare Document-19407. 

752 

753 The edges as defined by the kernel are not corrected because they 

754 have spurious values due to the convolution. 

755 """ 

756 image = exposure.getMaskedImage().getImage() 

757 

758 # The image needs to be units of electrons/holes 

759 with gainContext(exposure, image, applyGain, gains): 

760 

761 kLx = numpy.shape(kernel)[0] 

762 kLy = numpy.shape(kernel)[1] 

763 kernelImage = afwImage.ImageD(kLx, kLy) 

764 kernelImage.getArray()[:, :] = kernel 

765 tempImage = image.clone() 

766 

767 nanIndex = numpy.isnan(tempImage.getArray()) 

768 tempImage.getArray()[nanIndex] = 0. 

769 

770 outImage = afwImage.ImageF(image.getDimensions()) 

771 corr = numpy.zeros_like(image.getArray()) 

772 prev_image = numpy.zeros_like(image.getArray()) 

773 convCntrl = afwMath.ConvolutionControl(False, True, 1) 

774 fixedKernel = afwMath.FixedKernel(kernelImage) 

775 

776 # Define boundary by convolution region. The region that the correction will be 

777 # calculated for is one fewer in each dimension because of the second derivative terms. 

778 # NOTE: these need to use integer math, as we're using start:end as numpy index ranges. 

779 startX = kLx//2 

780 endX = -kLx//2 

781 startY = kLy//2 

782 endY = -kLy//2 

783 

784 for iteration in range(maxIter): 

785 

786 afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl) 

787 tmpArray = tempImage.getArray() 

788 outArray = outImage.getArray() 

789 

790 with numpy.errstate(invalid="ignore", over="ignore"): 

791 # First derivative term 

792 gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX]) 

793 gradOut = numpy.gradient(outArray[startY:endY, startX:endX]) 

794 first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1] 

795 

796 # Second derivative term 

797 diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1] 

798 diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX] 

799 second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21) 

800 

801 corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second) 

802 

803 tmpArray[:, :] = image.getArray()[:, :] 

804 tmpArray[nanIndex] = 0. 

805 tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX] 

806 

807 if iteration > 0: 

808 diff = numpy.sum(numpy.abs(prev_image - tmpArray)) 

809 

810 if diff < threshold: 

811 break 

812 prev_image[:, :] = tmpArray[:, :] 

813 

814 image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \ 

815 corr[startY + 1:endY - 1, startX + 1:endX - 1] 

816 

817 return diff, iteration 

818 

819 

820@contextmanager 

821def gainContext(exp, image, apply, gains=None): 

822 """Context manager that applies and removes gain. 

823 

824 Parameters 

825 ---------- 

826 exp : `lsst.afw.image.Exposure` 

827 Exposure to apply/remove gain. 

828 image : `lsst.afw.image.Image` 

829 Image to apply/remove gain. 

830 apply : `Bool` 

831 If True, apply and remove the amplifier gain. 

832 gains : `dict` [`str`, `float`] 

833 A dictionary, keyed by amplifier name, of the gains to use. 

834 If gains is None, the nominal gains in the amplifier object are used. 

835 

836 Yields 

837 ------ 

838 exp : `lsst.afw.image.Exposure` 

839 Exposure with the gain applied. 

840 """ 

841 # check we have all of them if provided because mixing and matching would 

842 # be a real mess 

843 if gains and apply is True: 

844 ampNames = [amp.getName() for amp in exp.getDetector()] 

845 for ampName in ampNames: 

846 if ampName not in gains.keys(): 

847 raise RuntimeError(f"Gains provided to gain context, but no entry found for amp {ampName}") 

848 

849 if apply: 

850 ccd = exp.getDetector() 

851 for amp in ccd: 

852 sim = image.Factory(image, amp.getBBox()) 

853 if gains: 

854 gain = gains[amp.getName()] 

855 else: 

856 gain = amp.getGain() 

857 sim *= gain 

858 

859 try: 

860 yield exp 

861 finally: 

862 if apply: 

863 ccd = exp.getDetector() 

864 for amp in ccd: 

865 sim = image.Factory(image, amp.getBBox()) 

866 if gains: 

867 gain = gains[amp.getName()] 

868 else: 

869 gain = amp.getGain() 

870 sim /= gain 

871 

872 

873def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, 

874 sensorTransmission=None, atmosphereTransmission=None): 

875 """Attach a TransmissionCurve to an Exposure, given separate curves for 

876 different components. 

877 

878 Parameters 

879 ---------- 

880 exposure : `lsst.afw.image.Exposure` 

881 Exposure object to modify by attaching the product of all given 

882 ``TransmissionCurves`` in post-assembly trimmed detector coordinates. 

883 Must have a valid ``Detector`` attached that matches the detector 

884 associated with sensorTransmission. 

885 opticsTransmission : `lsst.afw.image.TransmissionCurve` 

886 A ``TransmissionCurve`` that represents the throughput of the optics, 

887 to be evaluated in focal-plane coordinates. 

888 filterTransmission : `lsst.afw.image.TransmissionCurve` 

889 A ``TransmissionCurve`` that represents the throughput of the filter 

890 itself, to be evaluated in focal-plane coordinates. 

891 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

892 A ``TransmissionCurve`` that represents the throughput of the sensor 

893 itself, to be evaluated in post-assembly trimmed detector coordinates. 

894 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

895 A ``TransmissionCurve`` that represents the throughput of the 

896 atmosphere, assumed to be spatially constant. 

897 

898 Returns 

899 ------- 

900 combined : `lsst.afw.image.TransmissionCurve` 

901 The TransmissionCurve attached to the exposure. 

902 

903 Notes 

904 ----- 

905 All ``TransmissionCurve`` arguments are optional; if none are provided, the 

906 attached ``TransmissionCurve`` will have unit transmission everywhere. 

907 """ 

908 combined = afwImage.TransmissionCurve.makeIdentity() 

909 if atmosphereTransmission is not None: 

910 combined *= atmosphereTransmission 

911 if opticsTransmission is not None: 

912 combined *= opticsTransmission 

913 if filterTransmission is not None: 

914 combined *= filterTransmission 

915 detector = exposure.getDetector() 

916 fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE, 

917 toSys=camGeom.PIXELS) 

918 combined = combined.transformedBy(fpToPix) 

919 if sensorTransmission is not None: 

920 combined *= sensorTransmission 

921 exposure.getInfo().setTransmissionCurve(combined) 

922 return combined 

923 

924 

925@deprecated(reason="Camera geometry-based SkyWcs are now set when reading raws. To be removed after v19.", 

926 category=FutureWarning) 

927def addDistortionModel(exposure, camera): 

928 """!Update the WCS in exposure with a distortion model based on camera 

929 geometry. 

930 

931 Parameters 

932 ---------- 

933 exposure : `lsst.afw.image.Exposure` 

934 Exposure to process. Must contain a Detector and WCS. The 

935 exposure is modified. 

936 camera : `lsst.afw.cameraGeom.Camera` 

937 Camera geometry. 

938 

939 Raises 

940 ------ 

941 RuntimeError 

942 Raised if ``exposure`` is lacking a Detector or WCS, or if 

943 ``camera`` is None. 

944 Notes 

945 ----- 

946 Add a model for optical distortion based on geometry found in ``camera`` 

947 and the ``exposure``'s detector. The raw input exposure is assumed 

948 have a TAN WCS that has no compensation for optical distortion. 

949 Two other possibilities are: 

950 - The raw input exposure already has a model for optical distortion, 

951 as is the case for raw DECam data. 

952 In that case you should set config.doAddDistortionModel False. 

953 - The raw input exposure has a model for distortion, but it has known 

954 deficiencies severe enough to be worth fixing (e.g. because they 

955 cause problems for fitting a better WCS). In that case you should 

956 override this method with a version suitable for your raw data. 

957 

958 """ 

959 wcs = exposure.getWcs() 

960 if wcs is None: 

961 raise RuntimeError("exposure has no WCS") 

962 if camera is None: 

963 raise RuntimeError("camera is None") 

964 detector = exposure.getDetector() 

965 if detector is None: 

966 raise RuntimeError("exposure has no Detector") 

967 pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE) 

968 focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE, 

969 camGeom.FIELD_ANGLE) 

970 distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle) 

971 exposure.setWcs(distortedWcs) 

972 

973 

974def applyGains(exposure, normalizeGains=False): 

975 """Scale an exposure by the amplifier gains. 

976 

977 Parameters 

978 ---------- 

979 exposure : `lsst.afw.image.Exposure` 

980 Exposure to process. The image is modified. 

981 normalizeGains : `Bool`, optional 

982 If True, then amplifiers are scaled to force the median of 

983 each amplifier to equal the median of those medians. 

984 """ 

985 ccd = exposure.getDetector() 

986 ccdImage = exposure.getMaskedImage() 

987 

988 medians = [] 

989 for amp in ccd: 

990 sim = ccdImage.Factory(ccdImage, amp.getBBox()) 

991 sim *= amp.getGain() 

992 

993 if normalizeGains: 

994 medians.append(numpy.median(sim.getImage().getArray())) 

995 

996 if normalizeGains: 

997 median = numpy.median(numpy.array(medians)) 

998 for index, amp in enumerate(ccd): 

999 sim = ccdImage.Factory(ccdImage, amp.getBBox()) 

1000 if medians[index] != 0.0: 

1001 sim *= median/medians[index] 

1002 

1003 

1004def widenSaturationTrails(mask): 

1005 """Grow the saturation trails by an amount dependent on the width of the trail. 

1006 

1007 Parameters 

1008 ---------- 

1009 mask : `lsst.afw.image.Mask` 

1010 Mask which will have the saturated areas grown. 

1011 """ 

1012 

1013 extraGrowDict = {} 

1014 for i in range(1, 6): 

1015 extraGrowDict[i] = 0 

1016 for i in range(6, 8): 

1017 extraGrowDict[i] = 1 

1018 for i in range(8, 10): 

1019 extraGrowDict[i] = 3 

1020 extraGrowMax = 4 

1021 

1022 if extraGrowMax <= 0: 

1023 return 

1024 

1025 saturatedBit = mask.getPlaneBitMask("SAT") 

1026 

1027 xmin, ymin = mask.getBBox().getMin() 

1028 width = mask.getWidth() 

1029 

1030 thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK) 

1031 fpList = afwDetection.FootprintSet(mask, thresh).getFootprints() 

1032 

1033 for fp in fpList: 

1034 for s in fp.getSpans(): 

1035 x0, x1 = s.getX0(), s.getX1() 

1036 

1037 extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax) 

1038 if extraGrow > 0: 

1039 y = s.getY() - ymin 

1040 x0 -= xmin + extraGrow 

1041 x1 -= xmin - extraGrow 

1042 

1043 if x0 < 0: 

1044 x0 = 0 

1045 if x1 >= width - 1: 

1046 x1 = width - 1 

1047 

1048 mask.array[y, x0:x1+1] |= saturatedBit 

1049 

1050 

1051def setBadRegions(exposure, badStatistic="MEDIAN"): 

1052 """Set all BAD areas of the chip to the average of the rest of the exposure 

1053 

1054 Parameters 

1055 ---------- 

1056 exposure : `lsst.afw.image.Exposure` 

1057 Exposure to mask. The exposure mask is modified. 

1058 badStatistic : `str`, optional 

1059 Statistic to use to generate the replacement value from the 

1060 image data. Allowed values are 'MEDIAN' or 'MEANCLIP'. 

1061 

1062 Returns 

1063 ------- 

1064 badPixelCount : scalar 

1065 Number of bad pixels masked. 

1066 badPixelValue : scalar 

1067 Value substituted for bad pixels. 

1068 

1069 Raises 

1070 ------ 

1071 RuntimeError 

1072 Raised if `badStatistic` is not an allowed value. 

1073 """ 

1074 if badStatistic == "MEDIAN": 

1075 statistic = afwMath.MEDIAN 

1076 elif badStatistic == "MEANCLIP": 

1077 statistic = afwMath.MEANCLIP 

1078 else: 

1079 raise RuntimeError("Impossible method %s of bad region correction" % badStatistic) 

1080 

1081 mi = exposure.getMaskedImage() 

1082 mask = mi.getMask() 

1083 BAD = mask.getPlaneBitMask("BAD") 

1084 INTRP = mask.getPlaneBitMask("INTRP") 

1085 

1086 sctrl = afwMath.StatisticsControl() 

1087 sctrl.setAndMask(BAD) 

1088 value = afwMath.makeStatistics(mi, statistic, sctrl).getValue() 

1089 

1090 maskArray = mask.getArray() 

1091 imageArray = mi.getImage().getArray() 

1092 badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0) 

1093 imageArray[:] = numpy.where(badPixels, value, imageArray) 

1094 

1095 return badPixels.sum(), value