Coverage for python/lsst/ip/isr/ampOffset.py: 14%

156 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-03 17:29 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"] 

23 

24import warnings 

25 

26import numpy as np 

27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics 

28from lsst.afw.table import SourceTable 

29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask 

30from lsst.pex.config import Config, ConfigurableField, Field 

31from lsst.pipe.base import Task, Struct 

32 

33 

34class AmpOffsetConfig(Config): 

35 """Configuration parameters for AmpOffsetTask.""" 

36 

37 def setDefaults(self): 

38 self.background.algorithm = "AKIMA_SPLINE" 

39 self.background.useApprox = False 

40 self.background.ignoredPixelMask = [ 

41 "BAD", 

42 "SAT", 

43 "INTRP", 

44 "CR", 

45 "EDGE", 

46 "DETECTED", 

47 "DETECTED_NEGATIVE", 

48 "SUSPECT", 

49 "NO_DATA", 

50 ] 

51 self.detection.reEstimateBackground = False 

52 

53 # This maintains existing behavior and test values after DM-39796. 

54 self.detection.thresholdType = "stdev" 

55 

56 ampEdgeInset = Field( 

57 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running " 

58 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.", 

59 dtype=int, 

60 default=5, 

61 ) 

62 ampEdgeWidth = Field( 

63 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.", 

64 dtype=int, 

65 default=64, 

66 ) 

67 ampEdgeMinFrac = Field( 

68 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be " 

69 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.", 

70 dtype=float, 

71 default=0.5, 

72 ) 

73 ampEdgeMaxOffset = Field( 

74 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the " 

75 "result will be discarded and therefore not used to determine amp pedestal corrections.", 

76 dtype=float, 

77 default=5.0, 

78 ) 

79 ampEdgeWindowFrac = Field( 

80 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average " 

81 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should " 

82 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the " 

83 "sliding window used in obs_subaru for compatibility with existing HSC data.", 

84 dtype=float, 

85 default=512 / 4176, 

86 ) 

87 doBackground = Field( 

88 doc="Estimate and subtract background prior to amp offset estimation?", 

89 dtype=bool, 

90 default=True, 

91 ) 

92 background = ConfigurableField( 

93 doc="An initial background estimation step run prior to amp offset calculation.", 

94 target=SubtractBackgroundTask, 

95 ) 

96 backgroundFractionSample = Field( 

97 doc="The fraction of the shorter side of the amplifier used for background binning.", 

98 dtype=float, 

99 default=1.0, 

100 ) 

101 doDetection = Field( 

102 doc="Detect sources and update cloned exposure prior to amp offset estimation?", 

103 dtype=bool, 

104 default=True, 

105 ) 

106 detection = ConfigurableField( 

107 doc="Source detection to add temporary detection footprints prior to amp offset calculation.", 

108 target=SourceDetectionTask, 

109 ) 

110 

111 

112class AmpOffsetTask(Task): 

113 """Calculate and apply amp offset corrections to an exposure.""" 

114 

115 ConfigClass = AmpOffsetConfig 

116 _DefaultName = "isrAmpOffset" 

117 

118 def __init__(self, *args, **kwargs): 

119 super().__init__(*args, **kwargs) 

120 # Always load background subtask, even if doBackground=False; 

121 # this allows for default plane bit masks to be defined. 

122 self.makeSubtask("background") 

123 if self.config.doDetection: 

124 self.makeSubtask("detection") 

125 # Initialize all of the instance variables here. 

126 self.shortAmpSide = 0 

127 

128 def run(self, exposure): 

129 """Calculate amp offset values, determine corrective pedestals for each 

130 amp, and update the input exposure in-place. 

131 

132 Parameters 

133 ---------- 

134 exposure: `lsst.afw.image.Exposure` 

135 Exposure to be corrected for amp offsets. 

136 """ 

137 

138 # Generate an exposure clone to work on and establish the bit mask. 

139 exp = exposure.clone() 

140 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask) 

141 amps = exp.getDetector().getAmplifiers() 

142 

143 # Check that all amps have the same gemotry. 

144 ampDims = [amp.getBBox().getDimensions() for amp in amps] 

145 if not all(dim == ampDims[0] for dim in ampDims): 

146 raise RuntimeError("All amps should have the same geometry.") 

147 

148 # Assuming all the amps have the same geometry. 

149 self.shortAmpSide = np.min(ampDims[0]) 

150 

151 # Fit and subtract background. 

152 if self.config.doBackground: 

153 maskedImage = exp.getMaskedImage() 

154 # Assuming all the detectors are the same. 

155 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1 

156 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1 

157 # This ensures that the `binSize` is as large as possible, 

158 # preventing background subtraction from inadvertently removing the 

159 # amp offset signature. Here it's set to the shorter dimension of 

160 # the amplifier by default (`backgroundFractionSample` = 1), which 

161 # seems reasonable. 

162 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY)) 

163 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle) 

164 maskedImage -= bgImage 

165 

166 # Detect sources and update cloned exposure mask planes in-place. 

167 if self.config.doDetection: 

168 schema = SourceTable.makeMinimalSchema() 

169 table = SourceTable.make(schema) 

170 # Detection sigma, used for smoothing and to grow detections, is 

171 # normally measured from the PSF of the exposure. As the PSF hasn't 

172 # been measured at this stage of processing, sigma is instead 

173 # set to an approximate value here (which should be sufficient). 

174 _ = self.detection.run(table=table, exposure=exp, sigma=2) 

175 

176 # Safety check: do any pixels remain for amp offset estimation? 

177 if (exp.mask.array & bitMask).all(): 

178 self.log.warning( 

179 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set " 

180 "to zero." 

181 ) 

182 pedestals = np.zeros(len(amps)) 

183 else: 

184 # Set up amp offset inputs. 

185 im = exp.image 

186 im.array[(exp.mask.array & bitMask) > 0] = np.nan 

187 

188 if self.config.ampEdgeWindowFrac > 1: 

189 raise RuntimeError( 

190 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the " 

191 "edge length exceeds 1. This leads to complications downstream, after convolution in " 

192 "the `getSideAmpOffset()` method. Please modify the `ampEdgeWindowFrac` value in the " 

193 "config to be 1 or less and rerun." 

194 ) 

195 

196 # Determine amplifier geometry. 

197 ampAreas = {amp.getBBox().getArea() for amp in amps} 

198 if len(ampAreas) > 1: 

199 raise NotImplementedError( 

200 "Amp offset correction is not yet implemented for detectors with differing amp sizes." 

201 ) 

202 

203 # Obtain association and offset matrices. 

204 A, sides = self.getAmpAssociations(amps) 

205 B = self.getAmpOffsets(im, amps, A, sides) 

206 

207 # If least-squares minimization fails, convert NaNs to zeroes, 

208 # ensuring that no values are erroneously added/subtracted. 

209 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0]) 

210 

211 metadata = exposure.getMetadata() 

212 for amp, pedestal in zip(amps, pedestals): 

213 ampIm = exposure.image[amp.getBBox()].array 

214 ampIm -= pedestal 

215 ampName = amp.getName() 

216 metadata.set( 

217 f"LSST ISR AMPOFFSET PEDESTAL {ampName}", 

218 float(pedestal), 

219 f"Pedestal level subtracted from amp {ampName}", 

220 ) 

221 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}") 

222 

223 return Struct(pedestals=pedestals) 

224 

225 def getAmpAssociations(self, amps): 

226 """Determine amp geometry and amp associations from a list of 

227 amplifiers. 

228 

229 Parse an input list of amplifiers to determine the layout of amps 

230 within a detector, and identify all amp sides (i.e., the 

231 horizontal and vertical junctions between amps). 

232 

233 Returns a matrix with a shape corresponding to the geometry of the amps 

234 in the detector. 

235 

236 Parameters 

237 ---------- 

238 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

239 List of amplifier objects used to deduce associations. 

240 

241 Returns 

242 ------- 

243 ampAssociations : `numpy.ndarray` 

244 An N x N matrix (N = number of amplifiers) that illustrates the 

245 connections between amplifiers within the detector layout. Each row 

246 and column index corresponds to the ampIds of a specific pair of 

247 amplifiers, and the matrix elements indicate their associations as 

248 follows: 

249 0: No association 

250 -1: Association exists (direction specified in the ampSides matrix) 

251 n >= 1: Diagonal elements indicate the number of neighboring 

252 amplifiers for the corresponding ampId==row==column number. 

253 

254 ampSides : `numpy.ndarray` 

255 An N x N matrix (N = the number of amplifiers) representing the amp 

256 side information corresponding to the `ampAssociations` 

257 matrix. The elements are integers defined as below: 

258 -1: No side due to no association or the same amp (diagonals) 

259 0: Side on the bottom 

260 1: Side on the right 

261 2: Side on the top 

262 3: Side on the left 

263 """ 

264 xCenters = [amp.getBBox().getCenterX() for amp in amps] 

265 yCenters = [amp.getBBox().getCenterY() for amp in amps] 

266 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1 

267 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1 

268 

269 nAmps = len(amps) 

270 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int) 

271 

272 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices): 

273 ampIds[yIndex, xIndex] = ampId 

274 

275 ampAssociations = np.zeros((nAmps, nAmps), dtype=int) 

276 ampSides = np.full_like(ampAssociations, -1) 

277 

278 for ampId in ampIds.ravel(): 

279 neighbors, sides = self.getNeighbors(ampIds, ampId) 

280 ampAssociations[ampId, neighbors] = -1 

281 ampSides[ampId, neighbors] = sides 

282 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum() 

283 

284 if ampAssociations.sum() != 0: 

285 raise RuntimeError("The `ampAssociations` array does not sum to zero.") 

286 

287 if not np.all(ampAssociations == ampAssociations.T): 

288 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.") 

289 

290 self.log.debug("amp associations:\n%s", ampAssociations) 

291 self.log.debug("amp sides:\n%s", ampSides) 

292 

293 return ampAssociations, ampSides 

294 

295 def getNeighbors(self, ampIds, ampId): 

296 """Get the neighbor amplifiers and their sides for a given 

297 amplifier. 

298 

299 Parameters 

300 ---------- 

301 ampIds : `numpy.ndarray` 

302 Matrix with amp side association information. 

303 ampId : `int` 

304 The amplifier ID for which neighbor amplifiers and side IDs 

305 are to be found. 

306 

307 Returns 

308 ------- 

309 neighbors : `list` [`int`] 

310 List of neighbor amplifier IDs. 

311 sides : `list` [`int`] 

312 List of side IDs, with each ID corresponding to its respective 

313 neighbor amplifier. 

314 """ 

315 m, n = ampIds.shape 

316 r, c = np.ravel(np.where(ampIds == ampId)) 

317 neighbors, sides = [], [] 

318 sideLookup = { 

319 0: (r + 1, c), 

320 1: (r, c + 1), 

321 2: (r - 1, c), 

322 3: (r, c - 1), 

323 } 

324 for side, (row, column) in sideLookup.items(): 

325 if 0 <= row < m and 0 <= column < n: 

326 neighbors.append(ampIds[row][column]) 

327 sides.append(side) 

328 return neighbors, sides 

329 

330 def getAmpOffsets(self, im, amps, associations, sides): 

331 """Calculate the amp offsets for all amplifiers. 

332 

333 Parameters 

334 ---------- 

335 im : `lsst.afw.image._image.ImageF` 

336 Amplifier image to extract data from. 

337 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

338 List of amplifier objects. 

339 associations : numpy.ndarray 

340 An N x N matrix containing amp association information, where N is 

341 the number of amplifiers. 

342 sides : numpy.ndarray 

343 An N x N matrix containing amp side information, where N is the 

344 number of amplifiers. 

345 

346 Returns 

347 ------- 

348 ampsOffsets : `numpy.ndarray` 

349 1D float array containing the calculated amp offsets for all 

350 amplifiers. 

351 """ 

352 ampsOffsets = np.zeros(len(amps)) 

353 ampsEdges = self.getAmpEdges(im, amps, sides) 

354 interfaceOffsetLookup = {} 

355 for ampId, ampAssociations in enumerate(associations): 

356 ampNeighbors = np.ravel(np.where(ampAssociations < 0)) 

357 for ampNeighbor in ampNeighbors: 

358 ampSide = sides[ampId][ampNeighbor] 

359 edgeA = ampsEdges[ampId][ampSide] 

360 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4] 

361 if ampId < ampNeighbor: 

362 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB) 

363 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset 

364 else: 

365 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"] 

366 ampsOffsets[ampId] += interfaceOffset 

367 return ampsOffsets 

368 

369 def getAmpEdges(self, im, amps, ampSides): 

370 """Calculate the amp edges for all amplifiers. 

371 

372 Parameters 

373 ---------- 

374 im : `lsst.afw.image._image.ImageF` 

375 Amplifier image to extract data from. 

376 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

377 List of amplifier objects. 

378 ampSides : `numpy.ndarray` 

379 An N x N matrix containing amp side information, where N is the 

380 number of amplifiers. 

381 

382 Returns 

383 ------- 

384 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]] 

385 A dictionary containing amp edge(s) for each amplifier, 

386 corresponding to one or more potential sides, where each edge is 

387 associated with a side. The outer dictionary has integer keys 

388 representing amplifier IDs, and the inner dictionary has integer 

389 keys representing side IDs for each amplifier and values that are 

390 1D arrays of floats representing the 1D medianified strips from the 

391 amp image, referred to as "amp edge": 

392 {ampID: {sideID: numpy.ndarray}, ...} 

393 """ 

394 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth 

395 ampEdges = {} 

396 slice_map = { 

397 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)), 

398 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)), 

399 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)), 

400 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)), 

401 } 

402 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)): 

403 ampEdges[ampId] = {} 

404 ampIm = im[amp.getBBox()].array 

405 # Loop over identified sides. 

406 for ampSide in ampSides: 

407 if ampSide < 0: 

408 continue 

409 strip = ampIm[slice_map[ampSide]] 

410 # Catch warnings to prevent all-NaN slice RuntimeWarning. 

411 with warnings.catch_warnings(): 

412 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered") 

413 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip 

414 return ampEdges 

415 

416 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB): 

417 """Calculate the amp offset for a given interface between two 

418 amplifiers. 

419 

420 Parameters 

421 ---------- 

422 ampIdA : int 

423 ID of the first amplifier. 

424 ampIdB : int 

425 ID of the second amplifier. 

426 edgeA : numpy.ndarray 

427 Amp edge for the first amplifier. 

428 edgeB : numpy.ndarray 

429 Amp edge for the second amplifier. 

430 

431 Returns 

432 ------- 

433 interfaceOffset : float 

434 The calculated amp offset value for the given interface between 

435 amps A and B. 

436 """ 

437 interfaceId = f"{ampIdA}{ampIdB}" 

438 sctrl = StatisticsControl() 

439 # NOTE: Taking the difference with the order below fixes the sign flip 

440 # in the B matrix. 

441 edgeDiff = edgeA - edgeB 

442 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff)) 

443 # Compute rolling averages. 

444 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same") 

445 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same") 

446 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None) 

447 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan 

448 # Take clipped mean of rolling average data as amp offset value. 

449 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue() 

450 # Perform a couple of do-no-harm safety checks: 

451 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac, 

452 # b) The absolute offset ADU value is < ampEdgeMaxOffset. 

453 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg)) 

454 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac 

455 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset 

456 if minFracFail or maxOffsetFail: 

457 interfaceOffset = 0 

458 self.log.warning( 

459 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold " 

460 f"({self.config.ampEdgeMinFrac}) or the absolute offset value exceeds the limit " 

461 f"({self.config.ampEdgeMaxOffset} ADU). Setting the interface offset to 0." 

462 ) 

463 self.log.debug( 

464 f"amp interface {interfaceId} : " 

465 f"viable edge difference frac = {ampEdgeGoodFrac}, " 

466 f"interface offset = {interfaceOffset:.3f}" 

467 ) 

468 return interfaceOffset