Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Measure intra-detector crosstalk coefficients. 

24""" 

25 

26__all__ = ["MeasureCrosstalkConfig", "MeasureCrosstalkTask"] 

27 

28 

29import itertools 

30import numpy as np 

31 

32from lsstDebug import getDebugFrame 

33from lsst.afw.detection import FootprintSet, Threshold 

34from lsst.afw.display import getDisplay 

35from lsst.daf.persistence.butlerExceptions import NoResults 

36from lsst.pex.config import Config, Field, ListField, ConfigurableField 

37from lsst.pipe.base import CmdLineTask, Struct 

38 

39from .crosstalk import CrosstalkCalib 

40from .calibType import IsrProvenance 

41from .isrTask import IsrTask 

42 

43 

44class MeasureCrosstalkConfig(Config): 

45 """Configuration for MeasureCrosstalkTask.""" 

46 isr = ConfigurableField( 

47 target=IsrTask, 

48 doc="Instrument signature removal task to use to process data." 

49 ) 

50 threshold = Field( 

51 dtype=float, 

52 default=30000, 

53 doc="Minimum level of source pixels for which to measure crosstalk." 

54 ) 

55 doRerunIsr = Field( 

56 dtype=bool, 

57 default=True, 

58 doc="Rerun the ISR, even if postISRCCD files are available?" 

59 ) 

60 badMask = ListField( 

61 dtype=str, 

62 default=["SAT", "BAD", "INTRP"], 

63 doc="Mask planes to ignore when identifying source pixels." 

64 ) 

65 rejIter = Field( 

66 dtype=int, 

67 default=3, 

68 doc="Number of rejection iterations for final coefficient calculation." 

69 ) 

70 rejSigma = Field( 

71 dtype=float, 

72 default=2.0, 

73 doc="Rejection threshold (sigma) for final coefficient calculation." 

74 ) 

75 isTrimmed = Field( 

76 dtype=bool, 

77 default=True, 

78 doc="Have the amplifiers been trimmed before measuring CT?" 

79 ) 

80 

81 def setDefaults(self): 

82 Config.setDefaults(self) 

83 # Set ISR processing to run up until we would be applying the CT 

84 # correction. Applying subsequent stages may corrupt the signal. 

85 self.isr.doWrite = False 

86 self.isr.doOverscan = True 

87 self.isr.doAssembleCcd = True 

88 self.isr.doBias = True 

89 self.isr.doVariance = False # This isn't used in the calculation below. 

90 self.isr.doLinearize = True # This is the last ISR step we need. 

91 self.isr.doCrosstalk = False 

92 self.isr.doBrighterFatter = False 

93 self.isr.doDark = False 

94 self.isr.doStrayLight = False 

95 self.isr.doFlat = False 

96 self.isr.doFringe = False 

97 self.isr.doApplyGains = False 

98 self.isr.doDefect = True # Masking helps remove spurious pixels. 

99 self.isr.doSaturationInterpolation = False 

100 self.isr.growSaturationFootprintSize = 0 # We want the saturation spillover: it's good signal. 

101 

102 

103class MeasureCrosstalkTask(CmdLineTask): 

104 """Measure intra-detector crosstalk. 

105 

106 Notes 

107 ----- 

108 The crosstalk this method measures assumes that when a bright 

109 pixel is found in one detector amplifier, all other detector 

110 amplifiers may see an increase in the same pixel location 

111 (relative to the readout amplifier) as these other pixels are read 

112 out at the same time. 

113 

114 After processing each input exposure through a limited set of ISR 

115 stages, bright unmasked pixels above the threshold are identified. 

116 The potential CT signal is found by taking the ratio of the 

117 appropriate background-subtracted pixel value on the other 

118 amplifiers to the input value on the source amplifier. If the 

119 source amplifier has a large number of bright pixels as well, the 

120 background level may be elevated, leading to poor ratio 

121 measurements. 

122 

123 The set of ratios found between each pair of amplifiers across all 

124 input exposures is then gathered to produce the final CT 

125 coefficients. The sigma-clipped mean and sigma are returned from 

126 these sets of ratios, with the coefficient to supply to the ISR 

127 CrosstalkTask() being the multiplicative inverse of these values. 

128 """ 

129 ConfigClass = MeasureCrosstalkConfig 

130 _DefaultName = "measureCrosstalk" 

131 

132 def __init__(self, *args, **kwargs): 

133 CmdLineTask.__init__(self, *args, **kwargs) 

134 self.makeSubtask("isr") 

135 self.calib = CrosstalkCalib() 

136 

137 @classmethod 

138 def _makeArgumentParser(cls): 

139 parser = super(MeasureCrosstalkTask, cls)._makeArgumentParser() 

140 parser.add_argument("--crosstalkName", 

141 help="Name for this set of crosstalk coefficients", default="Unknown") 

142 parser.add_argument("--outputFileName", 

143 help="Name of yaml file to which to write crosstalk coefficients") 

144 parser.add_argument("--dump-ratios", dest="dumpRatios", 

145 help="Name of pickle file to which to write crosstalk ratios") 

146 return parser 

147 

148 @classmethod 

149 def parseAndRun(cls, *args, **kwargs): 

150 """Collate crosstalk results from multiple exposures. 

151 

152 Process all input exposures through runDataRef, construct 

153 final measurements from the final list of results from each 

154 input, and persist the output calibration. 

155 

156 This method will be deprecated as part of DM-24760. 

157 

158 Returns 

159 ------- 

160 coeff : `numpy.ndarray` 

161 Crosstalk coefficients. 

162 coeffErr : `numpy.ndarray` 

163 Crosstalk coefficient errors. 

164 coeffNum : `numpy.ndarray` 

165 Number of pixels used for crosstalk measurement. 

166 calib : `lsst.ip.isr.CrosstalkCalib` 

167 Crosstalk object created from the measurements. 

168 

169 """ 

170 kwargs["doReturnResults"] = True 

171 results = super(MeasureCrosstalkTask, cls).parseAndRun(*args, **kwargs) 

172 task = cls(config=results.parsedCmd.config, log=results.parsedCmd.log) 

173 resultList = [rr.result for rr in results.resultList] 

174 if results.parsedCmd.dumpRatios: 

175 import pickle 

176 pickle.dump(resultList, open(results.parsedCmd.dumpRatios, "wb")) 

177 coeff, coeffErr, coeffNum = task.reduce(resultList) 

178 

179 calib = CrosstalkCalib() 

180 provenance = IsrProvenance() 

181 

182 calib.coeffs = coeff 

183 calib.coeffErr = coeffErr 

184 calib.coeffNum = coeffNum 

185 

186 outputFileName = results.parsedCmd.outputFileName 

187 if outputFileName is not None: 

188 butler = results.parsedCmd.butler 

189 dataId = results.parsedCmd.id.idList[0] 

190 

191 # Rework to use lsst.ip.isr.CrosstalkCalib. 

192 det = butler.get('raw', dataId).getDetector() 

193 calib._detectorName = det.getName() 

194 calib._detectorSerial = det.getSerial() 

195 calib.nAmp = len(det) 

196 calib.hasCrosstalk = True 

197 calib.writeText(outputFileName + ".yaml") 

198 

199 provenance.calibType = 'CROSSTALK' 

200 provenance._detectorName = det.getName() 

201 provenance.fromDataIds(results.parsedCmd.id.idList) 

202 provenance.writeText(outputFileName + '_prov.yaml') 

203 

204 return Struct( 

205 coeff=coeff, 

206 coeffErr=coeffErr, 

207 coeffNum=coeffNum, 

208 calib=calib, 

209 ) 

210 

211 def _getConfigName(self): 

212 """Disable config output.""" 

213 return None 

214 

215 def _getMetadataName(self): 

216 """Disable metdata output.""" 

217 return None 

218 

219 def runDataRef(self, dataRef): 

220 """Get crosstalk ratios for detector. 

221 

222 Parameters 

223 ---------- 

224 dataRef : `lsst.daf.peristence.ButlerDataRef` 

225 Data references for detectors to process. 

226 

227 Returns 

228 ------- 

229 ratios : `list` of `list` of `numpy.ndarray` 

230 A matrix of pixel arrays. 

231 """ 

232 exposure = None 

233 if not self.config.doRerunIsr: 

234 try: 

235 exposure = dataRef.get("postISRCCD") 

236 except NoResults: 

237 pass 

238 

239 if exposure is None: 

240 exposure = self.isr.runDataRef(dataRef).exposure 

241 

242 dataId = dataRef.dataId 

243 return self.run(exposure, dataId=dataId) 

244 

245 def run(self, exposure, dataId=None): 

246 """Extract and return cross talk ratios for an exposure. 

247 

248 Parameters 

249 ---------- 

250 exposure : `lsst.afw.image.Exposure` 

251 Image data to measure crosstalk ratios from. 

252 dataId : 

253 Optional data ID for the exposure to process; used for logging. 

254 

255 Returns 

256 ------- 

257 ratios : `list` of `list` of `numpy.ndarray` 

258 A matrix of pixel arrays. 

259 """ 

260 ratios = self.extractCrosstalkRatios(exposure) 

261 self.log.info("Extracted %d pixels from %s", 

262 sum(len(jj) for ii in ratios for jj in ii if jj is not None), dataId) 

263 return ratios 

264 

265 def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None): 

266 """Extract crosstalk ratios between different amplifiers. 

267 

268 For pixels above ``threshold``, we calculate the ratio between 

269 each background-subtracted target amp and the source amp. We 

270 return a list of ratios for each pixel for each target/source 

271 combination, as a matrix of lists. 

272 

273 Parameters 

274 ---------- 

275 exposure : `lsst.afw.image.Exposure` 

276 Exposure for which to measure crosstalk. 

277 threshold : `float`, optional 

278 Lower limit on pixels for which we measure crosstalk. 

279 badPixels : `list` of `str`, optional 

280 Mask planes indicating a pixel is bad. 

281 

282 Returns 

283 ------- 

284 ratios : `list` of `list` of `numpy.ndarray` 

285 A matrix of pixel arrays. ``ratios[i][j]`` is an array of 

286 the fraction of the ``j``-th amp present on the ``i``-th amp. 

287 The value is `None` for the diagonal elements. 

288 

289 Notes 

290 ----- 

291 This has been moved into MeasureCrosstalkTask to allow for easier 

292 debugging. 

293 

294 The lsstDebug.Info() method can be rewritten for __name__ = 

295 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

296 

297 debug.display['extract'] : `bool` 

298 Display the exposure under consideration, with the pixels used 

299 for crosstalk measurement indicated by the DETECTED mask plane. 

300 debug.display['pixels'] : `bool` 

301 Display a plot of the ratio calculated for each pixel used in this 

302 exposure, split by amplifier pairs. The median value is listed 

303 for reference. 

304 """ 

305 if threshold is None: 

306 threshold = self.config.threshold 

307 if badPixels is None: 

308 badPixels = list(self.config.badMask) 

309 

310 mi = exposure.getMaskedImage() 

311 FootprintSet(mi, Threshold(threshold), "DETECTED") 

312 detected = mi.getMask().getPlaneBitMask("DETECTED") 

313 bad = mi.getMask().getPlaneBitMask(badPixels) 

314 bg = self.calib.calculateBackground(mi, badPixels + ["DETECTED"]) 

315 

316 self.debugView('extract', exposure) 

317 

318 ccd = exposure.getDetector() 

319 ratios = [[None for iAmp in ccd] for jAmp in ccd] 

320 

321 for ii, iAmp in enumerate(ccd): 

322 iImage = mi[iAmp.getBBox()] 

323 iMask = iImage.mask.array 

324 select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(iImage.image.array) 

325 for jj, jAmp in enumerate(ccd): 

326 if ii == jj: 

327 continue 

328 jImage = self.calib.extractAmp(mi.image, jAmp, iAmp, isTrimmed=self.config.isTrimmed) 

329 ratios[jj][ii] = (jImage.array[select] - bg)/iImage.image.array[select] 

330 self.debugPixels('pixels', iImage.image.array[select], jImage.array[select] - bg, ii, jj) 

331 return ratios 

332 

333 def reduce(self, ratioList): 

334 """Combine ratios to produce crosstalk coefficients. 

335 

336 Parameters 

337 ---------- 

338 ratioList : `list` of `list` of `list` of `numpy.ndarray` 

339 A list of matrices of arrays; a list of results from 

340 `extractCrosstalkRatios`. 

341 

342 Returns 

343 ------- 

344 coeff : `numpy.ndarray` 

345 Crosstalk coefficients. 

346 coeffErr : `numpy.ndarray` 

347 Crosstalk coefficient errors. 

348 coeffNum : `numpy.ndarray` 

349 Number of pixels used for crosstalk measurement. 

350 

351 Raises 

352 ------ 

353 RuntimeError 

354 Raised if there is no crosstalk data available. 

355 

356 Notes 

357 ----- 

358 The lsstDebug.Info() method can be rewritten for __name__ = 

359 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

360 

361 debug.display['reduce'] : `bool` 

362 Display a histogram of the combined ratio measurements for 

363 a pair of source/target amplifiers from all input 

364 exposures/detectors. 

365 """ 

366 numAmps = None 

367 for rr in ratioList: 

368 if rr is None: 

369 continue 

370 

371 if numAmps is None: 

372 numAmps = len(rr) 

373 

374 assert len(rr) == numAmps 

375 assert all(len(xx) == numAmps for xx in rr) 

376 

377 if numAmps is None: 

378 raise RuntimeError("Unable to measure crosstalk signal for any amplifier") 

379 

380 ratios = [[None for jj in range(numAmps)] for ii in range(numAmps)] 

381 for ii, jj in itertools.product(range(numAmps), range(numAmps)): 

382 if ii == jj: 

383 result = [] 

384 else: 

385 values = [rr[ii][jj] for rr in ratioList] 

386 num = sum(len(vv) for vv in values) 

387 if num == 0: 

388 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

389 result = np.nan 

390 else: 

391 result = np.concatenate([vv for vv in values if len(vv) > 0]) 

392 ratios[ii][jj] = result 

393 self.debugRatios('reduce', ratios, ii, jj) 

394 coeff, coeffErr, coeffNum = self.measureCrosstalkCoefficients(ratios, self.config.rejIter, 

395 self.config.rejSigma) 

396 self.log.info("Coefficients:\n%s\n", coeff) 

397 self.log.info("Errors:\n%s\n", coeffErr) 

398 self.log.info("Numbers:\n%s\n", coeffNum) 

399 return coeff, coeffErr, coeffNum 

400 

401 def measureCrosstalkCoefficients(self, ratios, rejIter=3, rejSigma=2.0): 

402 """Measure crosstalk coefficients from the ratios. 

403 

404 Given a list of ratios for each target/source amp combination, 

405 we measure a sigma clipped mean and error. 

406 

407 The coefficient errors returned are the standard deviation of 

408 the final set of clipped input ratios. 

409 

410 Parameters 

411 ---------- 

412 ratios : `list` of `list` of `numpy.ndarray` 

413 Matrix of arrays of ratios. 

414 rejIter : `int` 

415 Number of rejection iterations. 

416 rejSigma : `float` 

417 Rejection threshold (sigma). 

418 

419 Returns 

420 ------- 

421 coeff : `numpy.ndarray` 

422 Crosstalk coefficients. 

423 coeffErr : `numpy.ndarray` 

424 Crosstalk coefficient errors. 

425 coeffNum : `numpy.ndarray` 

426 Number of pixels for each measurement. 

427 

428 Notes 

429 ----- 

430 This has been moved into MeasureCrosstalkTask to allow for easier 

431 debugging. 

432 

433 The lsstDebug.Info() method can be rewritten for __name__ = 

434 `lsst.ip.isr.measureCrosstalk`, and supports the parameters: 

435 

436 debug.display['measure'] : `bool` 

437 Display a histogram of the combined ratio measurements for 

438 a pair of source/target amplifiers from the final set of 

439 clipped input ratios. 

440 """ 

441 if rejIter is None: 

442 rejIter = self.config.rejIter 

443 if rejSigma is None: 

444 rejSigma = self.config.rejSigma 

445 

446 numAmps = len(ratios) 

447 assert all(len(rr) == numAmps for rr in ratios) 

448 

449 coeff = np.zeros((numAmps, numAmps)) 

450 coeffErr = np.zeros((numAmps, numAmps)) 

451 coeffNum = np.zeros((numAmps, numAmps), dtype=int) 

452 

453 for ii, jj in itertools.product(range(numAmps), range(numAmps)): 

454 if ii == jj: 

455 values = [0.0] 

456 else: 

457 values = np.array(ratios[ii][jj]) 

458 values = values[np.abs(values) < 1.0] # Discard unreasonable values 

459 

460 coeffNum[ii][jj] = len(values) 

461 

462 if len(values) == 0: 

463 self.log.warn("No values for matrix element %d,%d" % (ii, jj)) 

464 coeff[ii][jj] = np.nan 

465 coeffErr[ii][jj] = np.nan 

466 else: 

467 if ii != jj: 

468 for rej in range(rejIter): 

469 lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) 

470 sigma = 0.741*(hi - lo) 

471 good = np.abs(values - med) < rejSigma*sigma 

472 if good.sum() == len(good): 

473 break 

474 values = values[good] 

475 

476 coeff[ii][jj] = np.mean(values) 

477 coeffErr[ii][jj] = np.nan if coeffNum[ii][jj] == 1 else np.std(values) 

478 self.debugRatios('measure', ratios, ii, jj) 

479 

480 return coeff, coeffErr, coeffNum 

481 

482 def debugView(self, stepname, exposure): 

483 """Utility function to examine the image being processed. 

484 

485 Parameters 

486 ---------- 

487 stepname : `str` 

488 State of processing to view. 

489 exposure : `lsst.afw.image.Exposure` 

490 Exposure to view. 

491 """ 

492 frame = getDebugFrame(self._display, stepname) 

493 if frame: 

494 display = getDisplay(frame) 

495 display.scale('asinh', 'zscale') 

496 display.mtv(exposure) 

497 

498 prompt = "Press Enter to continue: " 

499 while True: 

500 ans = input(prompt).lower() 

501 if ans in ("", "c",): 

502 break 

503 

504 def debugPixels(self, stepname, pixelsIn, pixelsOut, i, j): 

505 """Utility function to examine the CT ratio pixel values. 

506 

507 Parameters 

508 ---------- 

509 stepname : `str` 

510 State of processing to view. 

511 pixelsIn : `np.ndarray` 

512 Pixel values from the potential crosstalk "source". 

513 pixelsOut : `np.ndarray` 

514 Pixel values from the potential crosstalk "victim". 

515 i : `int` 

516 Index of the source amplifier. 

517 j : `int` 

518 Index of the target amplifier. 

519 """ 

520 frame = getDebugFrame(self._display, stepname) 

521 if frame: 

522 if i == j or len(pixelsIn) == 0 or len(pixelsOut) < 1: 

523 pass 

524 import matplotlib.pyplot as plot 

525 figure = plot.figure(1) 

526 figure.clear() 

527 

528 axes = figure.add_axes((0.1, 0.1, 0.8, 0.8)) 

529 axes.plot(pixelsIn, pixelsOut / pixelsIn, 'k+') 

530 plot.xlabel("Source amplifier pixel value") 

531 plot.ylabel("Measured pixel ratio") 

532 plot.title("(Source %d -> Victim %d) median ratio: %f" % 

533 (i, j, np.median(pixelsOut / pixelsIn))) 

534 figure.show() 

535 

536 prompt = "Press Enter to continue: " 

537 while True: 

538 ans = input(prompt).lower() 

539 if ans in ("", "c",): 

540 break 

541 plot.close() 

542 

543 def debugRatios(self, stepname, ratios, i, j): 

544 """Utility function to examine the final CT ratio set. 

545 

546 Parameters 

547 ---------- 

548 stepname : `str` 

549 State of processing to view. 

550 ratios : `List` of `List` of `np.ndarray` 

551 Array of measured CT ratios, indexed by source/victim 

552 amplifier. 

553 i : `int` 

554 Index of the source amplifier. 

555 j : `int` 

556 Index of the target amplifier. 

557 """ 

558 frame = getDebugFrame(self._display, stepname) 

559 if frame: 

560 if i == j or ratios is None or len(ratios) < 1: 

561 pass 

562 

563 RR = ratios[i][j] 

564 if RR is None or len(RR) < 1: 

565 pass 

566 

567 value = np.mean(RR) 

568 

569 import matplotlib.pyplot as plot 

570 figure = plot.figure(1) 

571 figure.clear() 

572 plot.hist(x=RR, bins='auto', color='b', rwidth=0.9) 

573 plot.xlabel("Measured pixel ratio") 

574 plot.axvline(x=value, color="k") 

575 plot.title("(Source %d -> Victim %d) clipped mean ratio: %f" % (i, j, value)) 

576 figure.show() 

577 

578 prompt = "Press Enter to continue: " 

579 while True: 

580 ans = input(prompt).lower() 

581 if ans in ("", "c",): 

582 break 

583 plot.close()