Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22""" 

23Apply intra-detector crosstalk corrections 

24""" 

25import numpy as np 

26 

27import lsst.afw.math 

28import lsst.afw.table 

29import lsst.afw.detection 

30from lsst.pex.config import Config, Field, ChoiceField, ListField 

31from lsst.pipe.base import Task 

32 

33__all__ = ["CrosstalkConfig", "CrosstalkTask", "subtractCrosstalk", "writeCrosstalkCoeffs", 

34 "NullCrosstalkTask"] 

35 

36 

37class CrosstalkConfig(Config): 

38 """Configuration for intra-detector crosstalk removal.""" 

39 minPixelToMask = Field( 

40 dtype=float, 

41 doc="Set crosstalk mask plane for pixels over this value.", 

42 default=45000 

43 ) 

44 crosstalkMaskPlane = Field( 

45 dtype=str, 

46 doc="Name for crosstalk mask plane.", 

47 default="CROSSTALK" 

48 ) 

49 crosstalkBackgroundMethod = ChoiceField( 

50 dtype=str, 

51 doc="Type of background subtraction to use when applying correction.", 

52 default="None", 

53 allowed={ 

54 "None": "Do no background subtraction.", 

55 "AMP": "Subtract amplifier-by-amplifier background levels.", 

56 "DETECTOR": "Subtract detector level background." 

57 }, 

58 ) 

59 useConfigCoefficients = Field( 

60 dtype=bool, 

61 doc="Ignore the detector crosstalk information in favor of CrosstalkConfig values?", 

62 default=False, 

63 ) 

64 crosstalkValues = ListField( 

65 dtype=float, 

66 doc=("Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 " 

67 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. " 

68 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column " 

69 "vector [corr0 corr1 corr2 ...]^T."), 

70 default=[0.0], 

71 ) 

72 crosstalkShape = ListField( 

73 dtype=int, 

74 doc="Shape of the coefficient array. This should be equal to [nAmp, nAmp].", 

75 default=[1], 

76 ) 

77 

78 def getCrosstalk(self, detector=None): 

79 """Return a 2-D numpy array of crosstalk coefficients in the proper shape. 

80 

81 Parameters 

82 ---------- 

83 detector : `lsst.afw.cameraGeom.detector` 

84 Detector that is to be crosstalk corrected. 

85 

86 Returns 

87 ------- 

88 coeffs : `numpy.ndarray` 

89 Crosstalk coefficients that can be used to correct the detector. 

90 

91 Raises 

92 ------ 

93 RuntimeError 

94 Raised if no coefficients could be generated from this detector/configuration. 

95 """ 

96 if self.useConfigCoefficients is True: 

97 coeffs = np.array(self.crosstalkValues).reshape(self.crosstalkShape) 

98 if detector is not None: 

99 nAmp = len(detector) 

100 if coeffs.shape != (nAmp, nAmp): 

101 raise RuntimeError("Constructed crosstalk coeffients do not match detector shape. " + 

102 f"{coeffs.shape} {nAmp}") 

103 return coeffs 

104 elif detector is not None and detector.hasCrosstalk() is True: 

105 # Assume the detector defines itself consistently. 

106 return detector.getCrosstalk() 

107 else: 

108 raise RuntimeError("Attempted to correct crosstalk without crosstalk coefficients") 

109 

110 def hasCrosstalk(self, detector=None): 

111 """Return a boolean indicating if crosstalk coefficients exist. 

112 

113 Parameters 

114 ---------- 

115 detector : `lsst.afw.cameraGeom.detector` 

116 Detector that is to be crosstalk corrected. 

117 

118 Returns 

119 ------- 

120 hasCrosstalk : `bool` 

121 True if this detector/configuration has crosstalk coefficients defined. 

122 """ 

123 if self.useConfigCoefficients is True and self.crosstalkValues is not None: 

124 return True 

125 elif detector is not None and detector.hasCrosstalk() is True: 

126 return True 

127 else: 

128 return False 

129 

130 

131class CrosstalkTask(Task): 

132 """Apply intra-detector crosstalk correction.""" 

133 ConfigClass = CrosstalkConfig 

134 _DefaultName = 'isrCrosstalk' 

135 

136 def prepCrosstalk(self, dataRef): 

137 """Placeholder for crosstalk preparation method, e.g., for inter-detector crosstalk. 

138 

139 Parameters 

140 ---------- 

141 dataRef : `daf.persistence.butlerSubset.ButlerDataRef` 

142 Butler reference of the detector data to be processed. 

143 

144 See also 

145 -------- 

146 lsst.obs.decam.crosstalk.DecamCrosstalkTask.prepCrosstalk 

147 """ 

148 return 

149 

150 def run(self, exposure, crosstalkSources=None, isTrimmed=False): 

151 """Apply intra-detector crosstalk correction 

152 

153 Parameters 

154 ---------- 

155 exposure : `lsst.afw.image.Exposure` 

156 Exposure for which to remove crosstalk. 

157 crosstalkSources : `defaultdict`, optional 

158 Image data and crosstalk coefficients from other detectors/amps that are 

159 sources of crosstalk in exposure. 

160 The default for intra-detector crosstalk here is None. 

161 isTrimmed : `bool` 

162 The image is already trimmed. 

163 This should no longer be needed once DM-15409 is resolved. 

164 

165 Raises 

166 ------ 

167 RuntimeError 

168 Raised if called for a detector that does not have a 

169 crosstalk correction. 

170 TypeError 

171 Raised if crosstalkSources is not None 

172 and not a numpy array or a dictionary. 

173 """ 

174 if crosstalkSources is not None: 

175 if isinstance(crosstalkSources, np.ndarray): 

176 coeffs = crosstalkSources 

177 elif isinstance(crosstalkSources, dict): 

178 # Nested dictionary produced by `measureCrosstalk.py` 

179 # There are two keys first in the nested dictionary 

180 for fKey, fValue in crosstalkSources.items(): 

181 for sKey, sValue in fValue.items(): 

182 firstKey = fKey 

183 secondKey = sKey 

184 tempDict = crosstalkSources[firstKey][secondKey] 

185 coeffs = [] 

186 for thirdKey in tempDict: 

187 tempList = [] 

188 for fourthKey in tempDict[thirdKey]: 

189 value = tempDict[thirdKey][fourthKey] 

190 tempList.append(value) 

191 coeffs.append(tempList) 

192 coeffs = np.array(coeffs) 

193 else: 

194 raise TypeError("crosstalkSources not of the correct type: `np.array` or `dict`") 

195 else: 

196 detector = exposure.getDetector() 

197 if not self.config.hasCrosstalk(detector=detector): 

198 raise RuntimeError("Attempted to correct crosstalk without crosstalk coefficients") 

199 coeffs = self.config.getCrosstalk(detector=detector) 

200 

201 self.log.info("Applying crosstalk correction.") 

202 subtractCrosstalk(exposure, crosstalkCoeffs=coeffs, 

203 minPixelToMask=self.config.minPixelToMask, 

204 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed, 

205 backgroundMethod=self.config.crosstalkBackgroundMethod) 

206 

207 

208# Flips required to get the corner to the lower-left 

209# (an arbitrary choice; flips are relative, so the choice of reference here is not important) 

210X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False, lsst.afw.cameraGeom.ReadoutCorner.LR: True, 

211 lsst.afw.cameraGeom.ReadoutCorner.UL: False, lsst.afw.cameraGeom.ReadoutCorner.UR: True} 

212Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL: False, lsst.afw.cameraGeom.ReadoutCorner.LR: False, 

213 lsst.afw.cameraGeom.ReadoutCorner.UL: True, lsst.afw.cameraGeom.ReadoutCorner.UR: True} 

214 

215 

216class NullCrosstalkTask(CrosstalkTask): 

217 def run(self, exposure, crosstalkSources=None): 

218 self.log.info("Not performing any crosstalk correction") 

219 

220 

221def extractAmp(image, amp, corner, isTrimmed=False): 

222 """Return an image of the amp 

223 

224 The returned image will have the amp's readout corner in the 

225 nominated `corner`. 

226 

227 Parameters 

228 ---------- 

229 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage` 

230 Image containing the amplifier of interest. 

231 amp : `lsst.afw.table.AmpInfoRecord` 

232 Amplifier information. 

233 corner : `lsst.afw.table.ReadoutCorner` or `None` 

234 Corner in which to put the amp's readout corner, or `None` for 

235 no flipping. 

236 isTrimmed : `bool` 

237 The image is already trimmed. 

238 This should no longer be needed once DM-15409 is resolved. 

239 

240 Returns 

241 ------- 

242 output : `lsst.afw.image.Image` 

243 Image of the amplifier in the standard configuration. 

244 """ 

245 output = image[amp.getBBox() if isTrimmed else amp.getRawDataBBox()] 

246 ampCorner = amp.getReadoutCorner() 

247 # Flipping is necessary only if the desired configuration doesn't match what we currently have 

248 xFlip = X_FLIP[corner] ^ X_FLIP[ampCorner] 

249 yFlip = Y_FLIP[corner] ^ Y_FLIP[ampCorner] 

250 return lsst.afw.math.flipImage(output, xFlip, yFlip) 

251 

252 

253def calculateBackground(mi, badPixels=["BAD"]): 

254 """Calculate median background in image 

255 

256 Getting a great background model isn't important for crosstalk correction, 

257 since the crosstalk is at a low level. The median should be sufficient. 

258 

259 Parameters 

260 ---------- 

261 mi : `lsst.afw.image.MaskedImage` 

262 MaskedImage for which to measure background. 

263 badPixels : `list` of `str` 

264 Mask planes to ignore. 

265 

266 Returns 

267 ------- 

268 bg : `float` 

269 Median background level. 

270 """ 

271 mask = mi.getMask() 

272 stats = lsst.afw.math.StatisticsControl() 

273 stats.setAndMask(mask.getPlaneBitMask(badPixels)) 

274 return lsst.afw.math.makeStatistics(mi, lsst.afw.math.MEDIAN, stats).getValue() 

275 

276 

277def subtractCrosstalk(exposure, crosstalkCoeffs=None, 

278 badPixels=["BAD"], minPixelToMask=45000, 

279 crosstalkStr="CROSSTALK", isTrimmed=False, 

280 backgroundMethod="None"): 

281 """Subtract the intra-detector crosstalk from an exposure 

282 

283 We set the mask plane indicated by ``crosstalkStr`` in a target amplifier 

284 for pixels in a source amplifier that exceed `minPixelToMask`. Note that 

285 the correction is applied to all pixels in the amplifier, but only those 

286 that have a substantial crosstalk are masked with ``crosstalkStr``. 

287 

288 The uncorrected image is used as a template for correction. This is good 

289 enough if the crosstalk is small (e.g., coefficients < ~ 1e-3), but if it's 

290 larger you may want to iterate. 

291 

292 This method needs unittests (DM-18876), but such testing requires 

293 DM-18610 to allow the test detector to have the crosstalk 

294 parameters set. 

295 

296 Parameters 

297 ---------- 

298 exposure : `lsst.afw.image.Exposure` 

299 Exposure for which to subtract crosstalk. 

300 crosstalkCoeffs : `numpy.ndarray` 

301 Coefficients to use to correct crosstalk. 

302 badPixels : `list` of `str` 

303 Mask planes to ignore. 

304 minPixelToMask : `float` 

305 Minimum pixel value (relative to the background level) in 

306 source amplifier for which to set ``crosstalkStr`` mask plane 

307 in target amplifier. 

308 crosstalkStr : `str` 

309 Mask plane name for pixels greatly modified by crosstalk. 

310 isTrimmed : `bool` 

311 The image is already trimmed. 

312 This should no longer be needed once DM-15409 is resolved. 

313 backgroundMethod : `str` 

314 Method used to subtract the background. "AMP" uses 

315 amplifier-by-amplifier background levels, "DETECTOR" uses full 

316 exposure/maskedImage levels. Any other value results in no 

317 background subtraction. 

318 """ 

319 mi = exposure.getMaskedImage() 

320 mask = mi.getMask() 

321 

322 ccd = exposure.getDetector() 

323 numAmps = len(ccd) 

324 if crosstalkCoeffs is None: 

325 coeffs = ccd.getCrosstalk() 

326 else: 

327 coeffs = crosstalkCoeffs 

328 assert coeffs.shape == (numAmps, numAmps) 

329 

330 # Set background level based on the requested method. The 

331 # thresholdBackground holds the offset needed so that we only mask 

332 # pixels high relative to the background, not in an absolute 

333 # sense. 

334 thresholdBackground = calculateBackground(mi, badPixels) 

335 

336 backgrounds = [0.0 for amp in ccd] 

337 if backgroundMethod is None: 

338 pass 

339 elif backgroundMethod == "AMP": 

340 backgrounds = [calculateBackground(mi[amp.getBBox()], badPixels) for amp in ccd] 

341 elif backgroundMethod == "DETECTOR": 

342 backgrounds = [calculateBackground(mi, badPixels) for amp in ccd] 

343 

344 # Set the crosstalkStr bit for the bright pixels (those which will have significant crosstalk correction) 

345 crosstalkPlane = mask.addMaskPlane(crosstalkStr) 

346 footprints = lsst.afw.detection.FootprintSet(mi, lsst.afw.detection.Threshold(minPixelToMask + 

347 thresholdBackground)) 

348 footprints.setMask(mask, crosstalkStr) 

349 crosstalk = mask.getPlaneBitMask(crosstalkStr) 

350 

351 # Do pixel level crosstalk correction. 

352 subtrahend = mi.Factory(mi.getBBox()) 

353 subtrahend.set((0, 0, 0)) 

354 for ii, iAmp in enumerate(ccd): 

355 iImage = subtrahend[iAmp.getBBox() if isTrimmed else iAmp.getRawDataBBox()] 

356 for jj, jAmp in enumerate(ccd): 

357 if ii == jj: 

358 assert coeffs[ii, jj] == 0.0 

359 if coeffs[ii, jj] == 0.0: 

360 continue 

361 

362 jImage = extractAmp(mi, jAmp, iAmp.getReadoutCorner(), isTrimmed) 

363 jImage.getMask().getArray()[:] &= crosstalk # Remove all other masks 

364 jImage -= backgrounds[jj] 

365 

366 iImage.scaledPlus(coeffs[ii, jj], jImage) 

367 

368 # Set crosstalkStr bit only for those pixels that have been significantly modified (i.e., those 

369 # masked as such in 'subtrahend'), not necessarily those that are bright originally. 

370 mask.clearMaskPlane(crosstalkPlane) 

371 mi -= subtrahend # also sets crosstalkStr bit for bright pixels 

372 

373 

374def writeCrosstalkCoeffs(outputFileName, coeff, det=None, crosstalkName="Unknown", indent=2): 

375 """Write a yaml file containing the crosstalk coefficients 

376 

377 The coeff array is indexed by [i, j] where i and j are amplifiers 

378 corresponding to the amplifiers in det 

379 

380 Parameters 

381 ---------- 

382 outputFileName : `str` 

383 Name of output yaml file 

384 coeff : `numpy.array(namp, namp)` 

385 numpy array of coefficients 

386 det : `lsst.afw.cameraGeom.Detector` 

387 Used to provide the list of amplifier names; 

388 if None use ['0', '1', ...] 

389 ccdType : `str` 

390 Name of detector, used to index the yaml file 

391 If all detectors are identical could be the type (e.g. ITL) 

392 indent : `int` 

393 Indent width to use when writing the yaml file 

394 """ 

395 

396 if det is None: 

397 ampNames = [str(i) for i in range(coeff.shape[0])] 

398 else: 

399 ampNames = [a.getName() for a in det] 

400 

401 assert coeff.shape == (len(ampNames), len(ampNames)) 

402 

403 dIndent = indent 

404 indent = 0 

405 with open(outputFileName, "w") as fd: 

406 print(indent*" " + "crosstalk :", file=fd) 

407 indent += dIndent 

408 print(indent*" " + "%s :" % crosstalkName, file=fd) 

409 indent += dIndent 

410 

411 for i, ampNameI in enumerate(ampNames): 

412 print(indent*" " + "%s : {" % ampNameI, file=fd) 

413 indent += dIndent 

414 print(indent*" ", file=fd, end='') 

415 

416 for j, ampNameJ in enumerate(ampNames): 

417 print("%s : %11.4e, " % (ampNameJ, coeff[i, j]), file=fd, 

418 end='\n' + indent*" " if j%4 == 3 else '') 

419 print("}", file=fd) 

420 

421 indent -= dIndent