Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy 

23 

24import lsst.geom 

25import lsst.afw.image as afwImage 

26import lsst.afw.math as afwMath 

27import lsst.afw.display as afwDisplay 

28 

29from lsst.pipe.base import Task, Struct, timeMethod 

30from lsst.pex.config import Config, Field, ListField, ConfigField 

31 

32afwDisplay.setDefaultMaskTransparency(75) 

33 

34 

35def getFrame(): 

36 """Produce a new frame number each time""" 

37 getFrame.frame += 1 

38 return getFrame.frame 

39 

40 

41getFrame.frame = 0 

42 

43 

44class FringeStatisticsConfig(Config): 

45 """Options for measuring fringes on an exposure""" 

46 badMaskPlanes = ListField(dtype=str, default=["SAT"], doc="Ignore pixels with these masks") 

47 stat = Field(dtype=int, default=int(afwMath.MEDIAN), doc="Statistic to use") 

48 clip = Field(dtype=float, default=3.0, doc="Sigma clip threshold") 

49 iterations = Field(dtype=int, default=3, doc="Number of fitting iterations") 

50 rngSeedOffset = Field(dtype=int, default=0, 

51 doc="Offset to the random number generator seed (full seed includes exposure ID)") 

52 

53 

54class FringeConfig(Config): 

55 """Fringe subtraction options""" 

56 filters = ListField(dtype=str, default=[], doc="Only fringe-subtract these filters") 

57 num = Field(dtype=int, default=30000, doc="Number of fringe measurements") 

58 small = Field(dtype=int, default=3, doc="Half-size of small (fringe) measurements (pixels)") 

59 large = Field(dtype=int, default=30, doc="Half-size of large (background) measurements (pixels)") 

60 iterations = Field(dtype=int, default=20, doc="Number of fitting iterations") 

61 clip = Field(dtype=float, default=3.0, doc="Sigma clip threshold") 

62 stats = ConfigField(dtype=FringeStatisticsConfig, doc="Statistics for measuring fringes") 

63 pedestal = Field(dtype=bool, default=False, doc="Remove fringe pedestal?") 

64 

65 

66class FringeTask(Task): 

67 """Task to remove fringes from a science exposure 

68 

69 We measure fringe amplitudes at random positions on the science exposure 

70 and at the same positions on the (potentially multiple) fringe frames 

71 and solve for the scales simultaneously. 

72 """ 

73 ConfigClass = FringeConfig 

74 _DefaultName = 'isrFringe' 

75 

76 def readFringes(self, dataRef, assembler=None): 

77 """Read the fringe frame(s), and pack data into a Struct 

78 

79 The current implementation assumes only a single fringe frame and 

80 will have to be updated to support multi-mode fringe subtraction. 

81 

82 This implementation could be optimised by persisting the fringe 

83 positions and fluxes. 

84 

85 Parameters 

86 ---------- 

87 dataRef : `daf.butler.butlerSubset.ButlerDataRef` 

88 Butler reference for the exposure that will have fringing 

89 removed. 

90 assembler : `lsst.ip.isr.AssembleCcdTask`, optional 

91 An instance of AssembleCcdTask (for assembling fringe 

92 frames). 

93 

94 Returns 

95 ------- 

96 fringeData : `pipeBase.Struct` 

97 Struct containing fringe data: 

98 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof 

99 Calibration fringe files containing master fringe frames. 

100 - ``seed`` : `int`, optional 

101 Seed for random number generation. 

102 """ 

103 try: 

104 fringe = dataRef.get("fringe", immediate=True) 

105 except Exception as e: 

106 raise RuntimeError("Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e)) 

107 

108 return self.loadFringes(fringe, assembler) 

109 

110 def loadFringes(self, fringeExp, expId=0, assembler=None): 

111 """Pack the fringe data into a Struct. 

112 

113 This method moves the struct parsing code into a butler 

114 generation agnostic handler. 

115 

116 Parameters 

117 ---------- 

118 fringeExp : `lsst.afw.exposure.Exposure` 

119 The exposure containing the fringe data. 

120 expId : `int`, optional 

121 Exposure id to be fringe corrected, used to set RNG seed. 

122 assembler : `lsst.ip.isr.AssembleCcdTask`, optional 

123 An instance of AssembleCcdTask (for assembling fringe 

124 frames). 

125 

126 Returns 

127 ------- 

128 fringeData : `pipeBase.Struct` 

129 Struct containing fringe data: 

130 - ``fringes`` : `lsst.afw.image.Exposure` or `list` thereof 

131 Calibration fringe files containing master fringe frames. 

132 - ``seed`` : `int`, optional 

133 Seed for random number generation. 

134 """ 

135 if assembler is not None: 

136 fringeExp = assembler.assembleCcd(fringeExp) 

137 

138 if expId is None: 

139 seed = self.config.stats.rngSeedOffset 

140 else: 

141 print(f"{self.config.stats.rngSeedOffset} {expId}") 

142 seed = self.config.stats.rngSeedOffset + expId 

143 

144 # Seed for numpy.random.RandomState must be convertable to a 32 bit unsigned integer 

145 seed %= 2**32 

146 

147 return Struct(fringes=fringeExp, 

148 seed=seed) 

149 

150 @timeMethod 

151 def run(self, exposure, fringes, seed=None): 

152 """Remove fringes from the provided science exposure. 

153 

154 Primary method of FringeTask. Fringes are only subtracted if the 

155 science exposure has a filter listed in the configuration. 

156 

157 Parameters 

158 ---------- 

159 exposure : `lsst.afw.image.Exposure` 

160 Science exposure from which to remove fringes. 

161 fringes : `lsst.afw.image.Exposure` or `list` thereof 

162 Calibration fringe files containing master fringe frames. 

163 seed : `int`, optional 

164 Seed for random number generation. 

165 

166 Returns 

167 ------- 

168 solution : `np.array` 

169 Fringe solution amplitudes for each input fringe frame. 

170 rms : `float` 

171 RMS error for the fit solution for this exposure. 

172 """ 

173 import lsstDebug 

174 display = lsstDebug.Info(__name__).display 

175 

176 if not self.checkFilter(exposure): 

177 self.log.info("Filter not found in FringeTaskConfig.filters. Skipping fringe correction.") 

178 return 

179 

180 if seed is None: 

181 seed = self.config.stats.rngSeedOffset 

182 rng = numpy.random.RandomState(seed=seed) 

183 

184 if not hasattr(fringes, '__iter__'): 

185 fringes = [fringes] 

186 

187 mask = exposure.getMaskedImage().getMask() 

188 for fringe in fringes: 

189 fringe.getMaskedImage().getMask().__ior__(mask) 

190 if self.config.pedestal: 

191 self.removePedestal(fringe) 

192 

193 positions = self.generatePositions(fringes[0], rng) 

194 fluxes = numpy.ndarray([self.config.num, len(fringes)]) 

195 for i, f in enumerate(fringes): 

196 fluxes[:, i] = self.measureExposure(f, positions, title="Fringe frame") 

197 

198 expFringes = self.measureExposure(exposure, positions, title="Science") 

199 solution, rms = self.solve(expFringes, fluxes) 

200 self.subtract(exposure, fringes, solution) 

201 if display: 

202 afwDisplay.Display(frame=getFrame()).mtv(exposure, title="Fringe subtracted") 

203 return solution, rms 

204 

205 @timeMethod 

206 def runDataRef(self, exposure, dataRef, assembler=None): 

207 """Remove fringes from the provided science exposure. 

208 

209 Retrieve fringes from butler dataRef provided and remove from 

210 provided science exposure. Fringes are only subtracted if the 

211 science exposure has a filter listed in the configuration. 

212 

213 Parameters 

214 ---------- 

215 exposure : `lsst.afw.image.Exposure` 

216 Science exposure from which to remove fringes. 

217 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

218 Butler reference to the exposure. Used to find 

219 appropriate fringe data. 

220 assembler : `lsst.ip.isr.AssembleCcdTask`, optional 

221 An instance of AssembleCcdTask (for assembling fringe 

222 frames). 

223 

224 Returns 

225 ------- 

226 solution : `np.array` 

227 Fringe solution amplitudes for each input fringe frame. 

228 rms : `float` 

229 RMS error for the fit solution for this exposure. 

230 """ 

231 if not self.checkFilter(exposure): 

232 self.log.info("Filter not found in FringeTaskConfig.filters. Skipping fringe correction.") 

233 return 

234 fringeStruct = self.readFringes(dataRef, assembler=assembler) 

235 return self.run(exposure, **fringeStruct.getDict()) 

236 

237 def checkFilter(self, exposure): 

238 """Check whether we should fringe-subtract the science exposure. 

239 

240 Parameters 

241 ---------- 

242 exposure : `lsst.afw.image.Exposure` 

243 Exposure to check the filter of. 

244 

245 Returns 

246 ------- 

247 needsFringe : `bool` 

248 If True, then the exposure has a filter listed in the 

249 configuration, and should have the fringe applied. 

250 """ 

251 return exposure.getFilter().getName() in self.config.filters 

252 

253 def removePedestal(self, fringe): 

254 """Remove pedestal from fringe exposure. 

255 

256 Parameters 

257 ---------- 

258 fringe : `lsst.afw.image.Exposure` 

259 Fringe data to subtract the pedestal value from. 

260 """ 

261 stats = afwMath.StatisticsControl() 

262 stats.setNumSigmaClip(self.config.stats.clip) 

263 stats.setNumIter(self.config.stats.iterations) 

264 mi = fringe.getMaskedImage() 

265 pedestal = afwMath.makeStatistics(mi, afwMath.MEDIAN, stats).getValue() 

266 self.log.info("Removing fringe pedestal: %f", pedestal) 

267 mi -= pedestal 

268 

269 def generatePositions(self, exposure, rng): 

270 """Generate a random distribution of positions for measuring fringe amplitudes. 

271 

272 Parameters 

273 ---------- 

274 exposure : `lsst.afw.image.Exposure` 

275 Exposure to measure the positions on. 

276 rng : `numpy.random.RandomState` 

277 Random number generator to use. 

278 

279 Returns 

280 ------- 

281 positions : `numpy.array` 

282 Two-dimensional array containing the positions to sample 

283 for fringe amplitudes. 

284 """ 

285 start = self.config.large 

286 num = self.config.num 

287 width = exposure.getWidth() - self.config.large 

288 height = exposure.getHeight() - self.config.large 

289 return numpy.array([rng.randint(start, width, size=num), 

290 rng.randint(start, height, size=num)]).swapaxes(0, 1) 

291 

292 @timeMethod 

293 def measureExposure(self, exposure, positions, title="Fringe"): 

294 """Measure fringe amplitudes for an exposure 

295 

296 The fringe amplitudes are measured as the statistic within a square 

297 aperture. The statistic within a larger aperture are subtracted so 

298 as to remove the background. 

299 

300 Parameters 

301 ---------- 

302 exposure : `lsst.afw.image.Exposure` 

303 Exposure to measure the positions on. 

304 positions : `numpy.array` 

305 Two-dimensional array containing the positions to sample 

306 for fringe amplitudes. 

307 title : `str`, optional 

308 Title used for debug out plots. 

309 

310 Returns 

311 ------- 

312 fringes : `numpy.array` 

313 Array of measured exposure values at each of the positions 

314 supplied. 

315 """ 

316 stats = afwMath.StatisticsControl() 

317 stats.setNumSigmaClip(self.config.stats.clip) 

318 stats.setNumIter(self.config.stats.iterations) 

319 stats.setAndMask(exposure.getMaskedImage().getMask().getPlaneBitMask(self.config.stats.badMaskPlanes)) 

320 

321 num = self.config.num 

322 fringes = numpy.ndarray(num) 

323 

324 for i in range(num): 

325 x, y = positions[i] 

326 small = measure(exposure.getMaskedImage(), x, y, self.config.small, self.config.stats.stat, stats) 

327 large = measure(exposure.getMaskedImage(), x, y, self.config.large, self.config.stats.stat, stats) 

328 fringes[i] = small - large 

329 

330 import lsstDebug 

331 display = lsstDebug.Info(__name__).display 

332 if display: 

333 disp = afwDisplay.Display(frame=getFrame()) 

334 disp.mtv(exposure, title=title) 

335 if False: 

336 with disp.Buffering(): 

337 for x, y in positions: 

338 corners = numpy.array([[-1, -1], [1, -1], [1, 1], [-1, 1], [-1, -1]]) + [[x, y]] 

339 disp.line(corners*self.config.small, ctype=afwDisplay.GREEN) 

340 disp.line(corners*self.config.large, ctype=afwDisplay.BLUE) 

341 

342 return fringes 

343 

344 @timeMethod 

345 def solve(self, science, fringes): 

346 """Solve for the scale factors with iterative clipping. 

347 

348 Parameters 

349 ---------- 

350 science : `numpy.array` 

351 Array of measured science image values at each of the 

352 positions supplied. 

353 fringes : `numpy.array` 

354 Array of measured fringe values at each of the positions 

355 supplied. 

356 

357 Returns 

358 ------- 

359 solution : `np.array` 

360 Fringe solution amplitudes for each input fringe frame. 

361 rms : `float` 

362 RMS error for the fit solution for this exposure. 

363 """ 

364 import lsstDebug 

365 doPlot = lsstDebug.Info(__name__).plot 

366 

367 origNum = len(science) 

368 

369 def emptyResult(msg=""): 

370 """Generate an empty result for return to the user 

371 

372 There are no good pixels; doesn't matter what we return. 

373 """ 

374 self.log.warn("Unable to solve for fringes: no good pixels%s", msg) 

375 out = [0] 

376 if len(fringes) > 1: 

377 out = out*len(fringes) 

378 return numpy.array(out), numpy.nan 

379 

380 good = numpy.where(numpy.logical_and(numpy.isfinite(science), numpy.any(numpy.isfinite(fringes), 1))) 

381 science = science[good] 

382 fringes = fringes[good] 

383 oldNum = len(science) 

384 if oldNum == 0: 

385 return emptyResult() 

386 

387 # Up-front rejection to get rid of extreme, potentially troublesome values 

388 # (e.g., fringe apertures that fall on objects). 

389 good = select(science, self.config.clip) 

390 for ff in range(fringes.shape[1]): 

391 good &= select(fringes[:, ff], self.config.clip) 

392 science = science[good] 

393 fringes = fringes[good] 

394 oldNum = len(science) 

395 if oldNum == 0: 

396 return emptyResult(" after initial rejection") 

397 

398 for i in range(self.config.iterations): 

399 solution = self._solve(science, fringes) 

400 resid = science - numpy.sum(solution*fringes, 1) 

401 rms = stdev(resid) 

402 good = numpy.logical_not(abs(resid) > self.config.clip*rms) 

403 self.log.debug("Iteration %d: RMS=%f numGood=%d", i, rms, good.sum()) 

404 self.log.debug("Solution %d: %s", i, solution) 

405 newNum = good.sum() 

406 if newNum == 0: 

407 return emptyResult(" after %d rejection iterations" % i) 

408 

409 if doPlot: 

410 import matplotlib.pyplot as plot 

411 for j in range(fringes.shape[1]): 

412 fig = plot.figure(j) 

413 fig.clf() 

414 try: 

415 fig.canvas._tkcanvas._root().lift() # == Tk's raise 

416 except Exception: 

417 pass 

418 ax = fig.add_subplot(1, 1, 1) 

419 adjust = science.copy() 

420 others = set(range(fringes.shape[1])) 

421 others.discard(j) 

422 for k in others: 

423 adjust -= solution[k]*fringes[:, k] 

424 ax.plot(fringes[:, j], adjust, 'r.') 

425 xmin = fringes[:, j].min() 

426 xmax = fringes[:, j].max() 

427 ymin = solution[j]*xmin 

428 ymax = solution[j]*xmax 

429 ax.plot([xmin, xmax], [ymin, ymax], 'b-') 

430 ax.set_title("Fringe %d: %f" % (j, solution[j])) 

431 ax.set_xlabel("Fringe amplitude") 

432 ax.set_ylabel("Science amplitude") 

433 ax.set_autoscale_on(False) 

434 ax.set_xbound(lower=xmin, upper=xmax) 

435 ax.set_ybound(lower=ymin, upper=ymax) 

436 fig.show() 

437 while True: 

438 ans = input("Enter or c to continue [chp]").lower() 

439 if ans in ("", "c",): 

440 break 

441 if ans in ("p",): 

442 import pdb 

443 pdb.set_trace() 

444 elif ans in ("h", ): 

445 print("h[elp] c[ontinue] p[db]") 

446 

447 if newNum == oldNum: 

448 # Not gaining 

449 break 

450 oldNum = newNum 

451 good = numpy.where(good) 

452 science = science[good] 

453 fringes = fringes[good] 

454 

455 # Final solution without rejection 

456 solution = self._solve(science, fringes) 

457 self.log.info("Fringe solution: %s RMS: %f Good: %d/%d", solution, rms, len(science), origNum) 

458 return solution, rms 

459 

460 def _solve(self, science, fringes): 

461 """Solve for the scale factors. 

462 

463 Parameters 

464 ---------- 

465 science : `numpy.array` 

466 Array of measured science image values at each of the 

467 positions supplied. 

468 fringes : `numpy.array` 

469 Array of measured fringe values at each of the positions 

470 supplied. 

471 

472 Returns 

473 ------- 

474 solution : `np.array` 

475 Fringe solution amplitudes for each input fringe frame. 

476 """ 

477 return afwMath.LeastSquares.fromDesignMatrix(fringes, science, 

478 afwMath.LeastSquares.DIRECT_SVD).getSolution() 

479 

480 def subtract(self, science, fringes, solution): 

481 """Subtract the fringes. 

482 

483 Parameters 

484 ---------- 

485 science : `lsst.afw.image.Exposure` 

486 Science exposure from which to remove fringes. 

487 fringes : `lsst.afw.image.Exposure` or `list` thereof 

488 Calibration fringe files containing master fringe frames. 

489 solution : `np.array` 

490 Fringe solution amplitudes for each input fringe frame. 

491 

492 Raises 

493 ------ 

494 RuntimeError : 

495 Raised if the number of fringe frames does not match the 

496 number of measured amplitudes. 

497 """ 

498 if len(solution) != len(fringes): 

499 raise RuntimeError("Number of fringe frames (%s) != number of scale factors (%s)" % 

500 (len(fringes), len(solution))) 

501 

502 for s, f in zip(solution, fringes): 

503 science.getMaskedImage().scaledMinus(s, f.getMaskedImage()) 

504 

505 

506def measure(mi, x, y, size, statistic, stats): 

507 """Measure a statistic within an aperture 

508 

509 @param mi MaskedImage to measure 

510 @param x, y Center for aperture 

511 @param size Size of aperture 

512 @param statistic Statistic to measure 

513 @param stats StatisticsControl object 

514 @return Value of statistic within aperture 

515 """ 

516 bbox = lsst.geom.Box2I(lsst.geom.Point2I(int(x) - size, int(y - size)), 

517 lsst.geom.Extent2I(2*size, 2*size)) 

518 subImage = mi.Factory(mi, bbox, afwImage.LOCAL) 

519 return afwMath.makeStatistics(subImage, statistic, stats).getValue() 

520 

521 

522def stdev(vector): 

523 """Calculate a robust standard deviation of an array of values 

524 

525 @param vector Array of values 

526 @return Standard deviation 

527 """ 

528 q1, q3 = numpy.percentile(vector, (25, 75)) 

529 return 0.74*(q3 - q1) 

530 

531 

532def select(vector, clip): 

533 """Select values within 'clip' standard deviations of the median 

534 

535 Returns a boolean array. 

536 """ 

537 q1, q2, q3 = numpy.percentile(vector, (25, 50, 75)) 

538 return numpy.abs(vector - q2) < clip*0.74*(q3 - q1)