Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Measure intra-detector crosstalk coefficients. 

24""" 

25 

26__all__ = ["MeasureCrosstalkConfig", "MeasureCrosstalkTask"] 

27 

28 

29import itertools 

30import numpy as np 

31 

32from lsstDebug import getDebugFrame 

33from lsst.afw.detection import FootprintSet, Threshold 

34from lsst.afw.display import getDisplay 

35from lsst.daf.persistence.butlerExceptions import NoResults 

36from lsst.pex.config import Config, Field, ListField, ConfigurableField 

37from lsst.pipe.base import CmdLineTask, Struct 

38 

39from .crosstalk import calculateBackground, extractAmp, writeCrosstalkCoeffs 

40from .isrTask import IsrTask 

41 

42 

43class MeasureCrosstalkConfig(Config): 

44 """Configuration for MeasureCrosstalkTask.""" 

45 isr = ConfigurableField( 

46 target=IsrTask, 

47 doc="Instrument signature removal task to use to process data." 

48 ) 

49 threshold = Field( 

50 dtype=float, 

51 default=30000, 

52 doc="Minimum level of source pixels for which to measure crosstalk." 

53 ) 

54 doRerunIsr = Field( 

55 dtype=bool, 

56 default=True, 

57 doc="Rerun the ISR, even if postISRCCD files are available?" 

58 ) 

59 badMask = ListField( 

60 dtype=str, 

61 default=["SAT", "BAD", "INTRP"], 

62 doc="Mask planes to ignore when identifying source pixels." 

63 ) 

64 rejIter = Field( 

65 dtype=int, 

66 default=3, 

67 doc="Number of rejection iterations for final coefficient calculation." 

68 ) 

69 rejSigma = Field( 

70 dtype=float, 

71 default=2.0, 

72 doc="Rejection threshold (sigma) for final coefficient calculation." 

73 ) 

74 isTrimmed = Field( 

75 dtype=bool, 

76 default=True, 

77 doc="Have the amplifiers been trimmed before measuring CT?" 

78 ) 

79 

80 def setDefaults(self): 

81 Config.setDefaults(self) 

82 # Set ISR processing to run up until we would be applying the CT 

83 # correction. Applying subsequent stages may corrupt the signal. 

84 self.isr.doWrite = False 

85 self.isr.doOverscan = True 

86 self.isr.doAssembleCcd = True 

87 self.isr.doBias = True 

88 self.isr.doVariance = False # This isn't used in the calculation below. 

89 self.isr.doLinearize = True # This is the last ISR step we need. 

90 self.isr.doCrosstalk = False 

91 self.isr.doBrighterFatter = False 

92 self.isr.doDark = False 

93 self.isr.doStrayLight = False 

94 self.isr.doFlat = False 

95 self.isr.doFringe = False 

96 self.isr.doApplyGains = False 

97 self.isr.doDefect = True # Masking helps remove spurious pixels. 

98 self.isr.doSaturationInterpolation = False 

99 self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal. 

100 

101 

102class MeasureCrosstalkTask(CmdLineTask): 

103 """Measure intra-detector crosstalk. 

104 

105 Notes 

106 ----- 

107 The crosstalk this method measures assumes that when a bright 

108 pixel is found in one detector amplifier, all other detector 

109 amplifiers may see an increase in the same pixel location 

110 (relative to the readout amplifier) as these other pixels are read 

111 out at the same time. 

112 

113 After processing each input exposure through a limited set of ISR 

114 stages, bright unmasked pixels above the threshold are identified. 

115 The potential CT signal is found by taking the ratio of the 

116 appropriate background-subtracted pixel value on the other 

117 amplifiers to the input value on the source amplifier. If the 

118 source amplifier has a large number of bright pixels as well, the 

119 background level may be elevated, leading to poor ratio 

120 measurements. 

121 

122 The set of ratios found between each pair of amplifiers across all 

123 input exposures is then gathered to produce the final CT 

124 coefficients. The sigma-clipped mean and sigma are returned from 

125 these sets of ratios, with the coefficient to supply to the ISR 

126 CrosstalkTask() being the multiplicative inverse of these values. 

127 """ 

128 ConfigClass = MeasureCrosstalkConfig 

129 _DefaultName = "measureCrosstalk" 

130 

131 def __init__(self, *args, **kwargs): 

132 CmdLineTask.__init__(self, *args, **kwargs) 

133 self.makeSubtask("isr") 

134 

135 @classmethod 

136 def _makeArgumentParser(cls): 

137 parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser() 

138 parser.add_argument("--crosstalkName", 

139 help="Name for this set of crosstalk coefficients", default="Unknown") 

140 parser.add_argument("--outputFileName", 

141 help="Name of yaml file to which to write crosstalk coefficients") 

142 parser.add_argument("--dump-ratios", dest="dumpRatios", 

143 help="Name of pickle file to which to write crosstalk ratios") 

144 return parser 

145 

146 @classmethod 

147 def parseAndRun(cls, *args, **kwargs): 

148 """Implement scatter/gather 

149 

150 Returns 

151 ------- 

152 coeff : `numpy.ndarray` 

153 Crosstalk coefficients. 

154 coeffErr : `numpy.ndarray` 

155 Crosstalk coefficient errors. 

156 coeffNum : `numpy.ndarray` 

157 Number of pixels used for crosstalk measurement. 

158 """ 

159 kwargs["doReturnResults"] = True 

160 results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs) 

161 task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log) 

162 resultList = [rr.result for rr in results.resultList] 

163 if results.parsedCmd.dumpRatios: 

164 import pickle 

165 pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "wb")) 

166 coeff, coeffErr, coeffNum = task.reduce(resultList) 

167 

168 outputFileName = results.parsedCmd.outputFileName 

169 if outputFileName is not None: 

170 butler = results.parsedCmd.butler 

171 dataId = results.parsedCmd.id.idList[0] 

172 dataId["detector"] = butler.queryMetadata("raw", ["detector"], dataId)[0] 

173 

174 det = butler.get('raw', dataId).getDetector() 

175 writeCrosstalkCoeffs(outputFileName, coeff, det=det, 

176 crosstalkName=results.parsedCmd.crosstalkName, indent=2) 

177 

178 return Struct( 

179 coeff=coeff, 

180 coeffErr=coeffErr, 

181 coeffNum=coeffNum 

182 ) 

183 

184 def _getConfigName(self): 

185 """Disable config output.""" 

186 return None 

187 

188 def _getMetadataName(self): 

189 """Disable metdata output.""" 

190 return None 

191 

192 def runDataRef(self, dataRef): 

193 """Get crosstalk ratios for detector. 

194 

195 Parameters 

196 ---------- 

197 dataRef : `lsst.daf.peristence.ButlerDataRef` 

198 Data references for detectors to process. 

199 

200 Returns 

201 ------- 

202 ratios : `list` of `list` of `numpy.ndarray` 

203 A matrix of pixel arrays. 

204 """ 

205 exposure = None 

206 if not self.config.doRerunIsr: 

207 try: 

208 exposure = dataRef.get("postISRCCD") 

209 except NoResults: 

210 pass 

211 

212 if exposure is None: 

213 exposure = self.isr.runDataRef(dataRef).exposure 

214 

215 dataId = dataRef.dataId 

216 return self.run(exposure, dataId=dataId) 

217 

218 def run(self, exposure, dataId=None): 

219 """Extract and return cross talk ratios for an exposure. 

220 

221 Parameters 

222 ---------- 

223 exposure : `lsst.afw.image.Exposure` 

224 Image data to measure crosstalk ratios from. 

225 dataId : 

226 Optional data ID for the exposure to process; used for logging. 

227 

228 Returns 

229 ------- 

230 ratios : `list` of `list` of `numpy.ndarray` 

231 A matrix of pixel arrays. 

232 """ 

233 ratios = self.extractCrosstalkRatios(exposure) 

234 self.log.info("Extracted %d pixels from %s", 

235 sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataId) 

236 return ratios 

237 

238 def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None): 

239 """Extract crosstalk ratios between different amplifiers. 

240 

241 For pixels above ``threshold``, we calculate the ratio between 

242 each background-subtracted target amp and the source amp. We 

243 return a list of ratios for each pixel for each target/source 

244 combination, as a matrix of lists. 

245 

246 Parameters 

247 ---------- 

248 exposure : `lsst.afw.image.Exposure` 

249 Exposure for which to measure crosstalk. 

250 threshold : `float`, optional 

251 Lower limit on pixels for which we measure crosstalk. 

252 badPixels : `list` of `str`, optional 

253 Mask planes indicating a pixel is bad. 

254 

255 Returns 

256 ------- 

257 ratios : `list` of `list` of `numpy.ndarray` 

258 A matrix of pixel arrays. ``ratios[i][j]`` is an array of 

259 the fraction of the ``j``-th amp present on the ``i``-th amp. 

260 The value is `None` for the diagonal elements. 

261 

262 Notes 

263 ----- 

264 This has been moved into MeasureCrosstalkTask to allow for easier 

265 debugging. 

266 

267 The lsstDebug.Info() method can be rewritten for __name__ = 

268 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

269 

270 debug.display['extract'] : `bool` 

271 Display the exposure under consideration, with the pixels used 

272 for crosstalk measurement indicated by the DETECTED mask plane. 

273 debug.display['pixels'] : `bool` 

274 Display a plot of the ratio calculated for each pixel used in this 

275 exposure, split by amplifier pairs. The median value is listed 

276 for reference. 

277 """ 

278 if threshold is None: 

279 threshold = self.config.threshold 

280 if badPixels is None: 

281 badPixels = list(self.config.badMask) 

282 

283 mi = exposure.getMaskedImage() 

284 FootprintSet(mi, Threshold(threshold), "DETECTED") 

285 detected = mi.getMask().getPlaneBitMask("DETECTED") 

286 bad = mi.getMask().getPlaneBitMask(badPixels) 

287 bg = calculateBackground(mi, badPixels + ["DETECTED"]) 

288 

289 self.debugView('extract', exposure) 

290 

291 ccd = exposure.getDetector() 

292 ratios = [[None for iAmp in ccd] for jAmp in ccd] 

293 

294 for ii, iAmp in enumerate(ccd): 

295 iImage = mi[iAmp.getBBox()] 

296 iMask = iImage.mask.array 

297 select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array) 

298 for jj, jAmp in enumerate(ccd): 

299 if ii == jj: 

300 continue 

301 jImage = extractAmp(mi.image, jAmp, iAmp.getReadoutCorner(), isTrimmed=self.config.isTrimmed) 

302 ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select] 

303 self.debugPixels('pixels', iImage.image.array[select], jImage.array[select] - bg, ii, jj) 

304 return ratios 

305 

306 def reduce(self, ratioList): 

307 """Combine ratios to produce crosstalk coefficients. 

308 

309 Parameters 

310 ---------- 

311 ratioList : `list` of `list` of `list` of `numpy.ndarray` 

312 A list of matrices of arrays; a list of results from 

313 `extractCrosstalkRatios`. 

314 

315 Returns 

316 ------- 

317 coeff : `numpy.ndarray` 

318 Crosstalk coefficients. 

319 coeffErr : `numpy.ndarray` 

320 Crosstalk coefficient errors. 

321 coeffNum : `numpy.ndarray` 

322 Number of pixels used for crosstalk measurement. 

323 

324 Raises 

325 ------ 

326 RuntimeError 

327 Raised if there is no crosstalk data available. 

328 

329 Notes 

330 ----- 

331 The lsstDebug.Info() method can be rewritten for __name__ = 

332 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

333 

334 debug.display['reduce'] : `bool` 

335 Display a histogram of the combined ratio measurements for 

336 a pair of source/target amplifiers from all input 

337 exposures/detectors. 

338 """ 

339 numAmps = None 

340 for rr in ratioList: 

341 if rr is None: 

342 continue 

343 

344 if numAmps is None: 

345 numAmps = len(rr) 

346 

347 assert len(rr) == numAmps 

348 assert all(len(xx) == numAmps for xx in rr) 

349 

350 if numAmps is None: 

351 raise RuntimeError("Unable to measure crosstalk signal for any amplifier") 

352 

353 ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)] 

354 for ii, jj in itertools.product(range(numAmps), range(numAmps)): 

355 if ii == jj: 

356 result = [] 

357 else: 

358 values = [rr[ii][jj] for rr in ratioList] 

359 num = sum(len(vv) for vv in values) 

360 if num == 0: 

361 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

362 result = np.nan 

363 else: 

364 result = np.concatenate([vv for vv in values if len(vv) > 0]) 

365 ratios[ii][jj] = result 

366 self.debugRatios('reduce', ratios, ii, jj) 

367 coeff, coeffErr, coeffNum = self.measureCrosstalkCoefficients(ratios, self.config.rejIter, 

368 self.config.rejSigma) 

369 self.log.info("Coefficients:\n%s\n", coeff) 

370 self.log.info("Errors:\n%s\n", coeffErr) 

371 self.log.info("Numbers:\n%s\n", coeffNum) 

372 return coeff, coeffErr, coeffNum 

373 

374 def measureCrosstalkCoefficients(self, ratios, rejIter=3, rejSigma=2.0): 

375 """Measure crosstalk coefficients from the ratios. 

376 

377 Given a list of ratios for each target/source amp combination, 

378 we measure a sigma clipped mean and error. 

379 

380 The coefficient errors returned are the standard deviation of 

381 the final set of clipped input ratios. 

382 

383 Parameters 

384 ---------- 

385 ratios : `list` of `list` of `numpy.ndarray` 

386 Matrix of arrays of ratios. 

387 rejIter : `int` 

388 Number of rejection iterations. 

389 rejSigma : `float` 

390 Rejection threshold (sigma). 

391 

392 Returns 

393 ------- 

394 coeff : `numpy.ndarray` 

395 Crosstalk coefficients. 

396 coeffErr : `numpy.ndarray` 

397 Crosstalk coefficient errors. 

398 coeffNum : `numpy.ndarray` 

399 Number of pixels for each measurement. 

400 

401 Notes 

402 ----- 

403 This has been moved into MeasureCrosstalkTask to allow for easier 

404 debugging. 

405 

406 The lsstDebug.Info() method can be rewritten for __name__ = 

407 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

408 

409 debug.display['measure'] : `bool` 

410 Display a histogram of the combined ratio measurements for 

411 a pair of source/target amplifiers from the final set of 

412 clipped input ratios. 

413 """ 

414 if rejIter is None: 

415 rejIter = self.config.rejIter 

416 if rejSigma is None: 

417 rejSigma = self.config.rejSigma 

418 

419 numAmps = len(ratios) 

420 assert all(len(rr) == numAmps for rr in ratios) 

421 

422 coeff = np.zeros((numAmps, numAmps)) 

423 coeffErr = np.zeros((numAmps, numAmps)) 

424 coeffNum = np.zeros((numAmps, numAmps), dtype=int) 

425 

426 for ii, jj in itertools.product(range(numAmps), range(numAmps)): 

427 if ii == jj: 

428 values = [0.0] 

429 else: 

430 values = np.array(ratios[ii][jj]) 

431 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

432 

433 coeffNum[ii][jj] = len(values) 

434 

435 if len(values) == 0: 

436 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

437 coeff[ii][jj] = np.nan 

438 coeffErr[ii][jj] = np.nan 

439 else: 

440 if ii != jj: 

441 for rej in range(rejIter): 

442 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

443 sigma = 0.741*(hi - lo) 

444 good = np.abs(values - med) < rejSigma*sigma 

445 if good.sum() == len(good): 

446 break 

447 values = values[good] 

448 

449 coeff[ii][jj] = np.mean(values) 

450 coeffErr[ii][jj] = np.nan if coeffNum[ii][jj] == 1 else np.std(values) 

451 self.debugRatios('measure', ratios, ii, jj) 

452 

453 return coeff, coeffErr, coeffNum 

454 

455 def debugView(self, stepname, exposure): 

456 """Utility function to examine the image being processed. 

457 

458 Parameters 

459 ---------- 

460 stepname : `str` 

461 State of processing to view. 

462 exposure : `lsst.afw.image.Exposure` 

463 Exposure to view. 

464 """ 

465 frame = getDebugFrame(self._display, stepname) 

466 if frame: 

467 display = getDisplay(frame) 

468 display.scale('asinh', 'zscale') 

469 display.mtv(exposure) 

470 

471 prompt = "Press Enter to continue: " 

472 while True: 

473 ans = input(prompt).lower() 

474 if ans in ("", "c",): 

475 break 

476 

477 def debugPixels(self, stepname, pixelsIn, pixelsOut, i, j): 

478 """Utility function to examine the CT ratio pixel values. 

479 

480 Parameters 

481 ---------- 

482 stepname : `str` 

483 State of processing to view. 

484 pixelsIn : `np.ndarray` 

485 Pixel values from the potential crosstalk "source". 

486 pixelsOut : `np.ndarray` 

487 Pixel values from the potential crosstalk "victim". 

488 i : `int` 

489 Index of the source amplifier. 

490 j : `int` 

491 Index of the target amplifier. 

492 """ 

493 frame = getDebugFrame(self._display, stepname) 

494 if frame: 

495 if i == j or len(pixelsIn) == 0 or len(pixelsOut) < 1: 

496 pass 

497 import matplotlib.pyplot as plot 

498 figure = plot.figure(1) 

499 figure.clear() 

500 

501 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

502 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+') 

503 plot.xlabel("Source amplifier pixel value") 

504 plot.ylabel("Measured pixel ratio") 

505 plot.title("(Source %d -> Victim %d) median ratio: %f" % 

506 (i, j, np.median(pixelsOut / pixelsIn))) 

507 figure.show() 

508 

509 prompt = "Press Enter to continue: " 

510 while True: 

511 ans = input(prompt).lower() 

512 if ans in ("", "c",): 

513 break 

514 plot.close() 

515 

516 def debugRatios(self, stepname, ratios, i, j): 

517 """Utility function to examine the final CT ratio set. 

518 

519 Parameters 

520 ---------- 

521 stepname : `str` 

522 State of processing to view. 

523 ratios : `List` of `List` of `np.ndarray` 

524 Array of measured CT ratios, indexed by source/victim 

525 amplifier. 

526 i : `int` 

527 Index of the source amplifier. 

528 j : `int` 

529 Index of the target amplifier. 

530 """ 

531 frame = getDebugFrame(self._display, stepname) 

532 if frame: 

533 if i == j or ratios is None or len(ratios) < 1: 

534 pass 

535 

536 RR = ratios[i][j] 

537 if RR is None or len(RR) < 1: 

538 pass 

539 

540 value = np.mean(RR) 

541 

542 import matplotlib.pyplot as plot 

543 figure = plot.figure(1) 

544 figure.clear() 

545 plot.hist(x=RR, bins='auto', color='b', rwidth=0.9) 

546 plot.xlabel("Measured pixel ratio") 

547 plot.axvline(x=value, color="k") 

548 plot.title("(Source %d -> Victim %d) clipped mean ratio: %f" % (i, j, value)) 

549 figure.show() 

550 

551 prompt = "Press Enter to continue: " 

552 while True: 

553 ans = input(prompt).lower() 

554 if ans in ("", "c",): 

555 break 

556 plot.close()