Coverage for python/lsst/ip/isr/ampOffset.py: 14%

155 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-12 10:46 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"] 

23 

24import warnings 

25 

26import numpy as np 

27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics 

28from lsst.afw.table import SourceTable 

29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask 

30from lsst.pex.config import Config, ConfigurableField, Field 

31from lsst.pipe.base import Task, Struct 

32 

33 

34class AmpOffsetConfig(Config): 

35 """Configuration parameters for AmpOffsetTask.""" 

36 

37 def setDefaults(self): 

38 self.background.algorithm = "AKIMA_SPLINE" 

39 self.background.useApprox = False 

40 self.background.ignoredPixelMask = [ 

41 "BAD", 

42 "SAT", 

43 "INTRP", 

44 "CR", 

45 "EDGE", 

46 "DETECTED", 

47 "DETECTED_NEGATIVE", 

48 "SUSPECT", 

49 "NO_DATA", 

50 ] 

51 self.detection.reEstimateBackground = False 

52 

53 ampEdgeInset = Field( 

54 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running " 

55 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.", 

56 dtype=int, 

57 default=5, 

58 ) 

59 ampEdgeWidth = Field( 

60 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.", 

61 dtype=int, 

62 default=64, 

63 ) 

64 ampEdgeMinFrac = Field( 

65 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be " 

66 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.", 

67 dtype=float, 

68 default=0.5, 

69 ) 

70 ampEdgeMaxOffset = Field( 

71 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the " 

72 "result will be discarded and therefore not used to determine amp pedestal corrections.", 

73 dtype=float, 

74 default=5.0, 

75 ) 

76 ampEdgeWindowFrac = Field( 

77 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average " 

78 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should " 

79 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the " 

80 "sliding window used in obs_subaru for compatibility with existing HSC data.", 

81 dtype=float, 

82 default=512 / 4176, 

83 ) 

84 doBackground = Field( 

85 doc="Estimate and subtract background prior to amp offset estimation?", 

86 dtype=bool, 

87 default=True, 

88 ) 

89 background = ConfigurableField( 

90 doc="An initial background estimation step run prior to amp offset calculation.", 

91 target=SubtractBackgroundTask, 

92 ) 

93 backgroundFractionSample = Field( 

94 doc="The fraction of the shorter side of the amplifier used for background binning.", 

95 dtype=float, 

96 default=1.0, 

97 ) 

98 doDetection = Field( 

99 doc="Detect sources and update cloned exposure prior to amp offset estimation?", 

100 dtype=bool, 

101 default=True, 

102 ) 

103 detection = ConfigurableField( 

104 doc="Source detection to add temporary detection footprints prior to amp offset calculation.", 

105 target=SourceDetectionTask, 

106 ) 

107 

108 

109class AmpOffsetTask(Task): 

110 """Calculate and apply amp offset corrections to an exposure.""" 

111 

112 ConfigClass = AmpOffsetConfig 

113 _DefaultName = "isrAmpOffset" 

114 

115 def __init__(self, *args, **kwargs): 

116 super().__init__(*args, **kwargs) 

117 # Always load background subtask, even if doBackground=False; 

118 # this allows for default plane bit masks to be defined. 

119 self.makeSubtask("background") 

120 if self.config.doDetection: 

121 self.makeSubtask("detection") 

122 # Initialize all of the instance variables here. 

123 self.shortAmpSide = 0 

124 

125 def run(self, exposure): 

126 """Calculate amp offset values, determine corrective pedestals for each 

127 amp, and update the input exposure in-place. 

128 

129 Parameters 

130 ---------- 

131 exposure: `lsst.afw.image.Exposure` 

132 Exposure to be corrected for amp offsets. 

133 """ 

134 

135 # Generate an exposure clone to work on and establish the bit mask. 

136 exp = exposure.clone() 

137 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask) 

138 amps = exp.getDetector().getAmplifiers() 

139 

140 # Check that all amps have the same gemotry. 

141 ampDims = [amp.getBBox().getDimensions() for amp in amps] 

142 if not all(dim == ampDims[0] for dim in ampDims): 

143 raise RuntimeError("All amps should have the same geometry.") 

144 

145 # Assuming all the amps have the same geometry. 

146 self.shortAmpSide = np.min(ampDims[0]) 

147 

148 # Fit and subtract background. 

149 if self.config.doBackground: 

150 maskedImage = exp.getMaskedImage() 

151 # Assuming all the detectors are the same. 

152 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1 

153 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1 

154 # This ensures that the `binSize` is as large as possible, 

155 # preventing background subtraction from inadvertently removing the 

156 # amp offset signature. Here it's set to the shorter dimension of 

157 # the amplifier by default (`backgroundFractionSample` = 1), which 

158 # seems reasonable. 

159 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY)) 

160 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle) 

161 maskedImage -= bgImage 

162 

163 # Detect sources and update cloned exposure mask planes in-place. 

164 if self.config.doDetection: 

165 schema = SourceTable.makeMinimalSchema() 

166 table = SourceTable.make(schema) 

167 # Detection sigma, used for smoothing and to grow detections, is 

168 # normally measured from the PSF of the exposure. As the PSF hasn't 

169 # been measured at this stage of processing, sigma is instead 

170 # set to an approximate value here (which should be sufficient). 

171 _ = self.detection.run(table=table, exposure=exp, sigma=2) 

172 

173 # Safety check: do any pixels remain for amp offset estimation? 

174 if (exp.mask.array & bitMask).all(): 

175 self.log.warning( 

176 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set " 

177 "to zero." 

178 ) 

179 pedestals = np.zeros(len(amps)) 

180 else: 

181 # Set up amp offset inputs. 

182 im = exp.image 

183 im.array[(exp.mask.array & bitMask) > 0] = np.nan 

184 

185 if self.config.ampEdgeWindowFrac > 1: 

186 raise RuntimeError( 

187 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the " 

188 "edge length exceeds 1. This leads to complications downstream, after convolution in " 

189 "the `getSideAmpOffset()` method. Please modify the `ampEdgeWindowFrac` value in the " 

190 "config to be 1 or less and rerun." 

191 ) 

192 

193 # Determine amplifier geometry. 

194 ampAreas = {amp.getBBox().getArea() for amp in amps} 

195 if len(ampAreas) > 1: 

196 raise NotImplementedError( 

197 "Amp offset correction is not yet implemented for detectors with differing amp sizes." 

198 ) 

199 

200 # Obtain association and offset matrices. 

201 A, sides = self.getAmpAssociations(amps) 

202 B = self.getAmpOffsets(im, amps, A, sides) 

203 

204 # If least-squares minimization fails, convert NaNs to zeroes, 

205 # ensuring that no values are erroneously added/subtracted. 

206 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0]) 

207 

208 metadata = exposure.getMetadata() 

209 for amp, pedestal in zip(amps, pedestals): 

210 ampIm = exposure.image[amp.getBBox()].array 

211 ampIm -= pedestal 

212 ampName = amp.getName() 

213 metadata.set( 

214 f"LSST ISR AMPOFFSET PEDESTAL {ampName}", 

215 float(pedestal), 

216 f"Pedestal level subtracted from amp {ampName}", 

217 ) 

218 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}") 

219 

220 return Struct(pedestals=pedestals) 

221 

222 def getAmpAssociations(self, amps): 

223 """Determine amp geometry and amp associations from a list of 

224 amplifiers. 

225 

226 Parse an input list of amplifiers to determine the layout of amps 

227 within a detector, and identify all amp sides (i.e., the 

228 horizontal and vertical junctions between amps). 

229 

230 Returns a matrix with a shape corresponding to the geometry of the amps 

231 in the detector. 

232 

233 Parameters 

234 ---------- 

235 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

236 List of amplifier objects used to deduce associations. 

237 

238 Returns 

239 ------- 

240 ampAssociations : `numpy.ndarray` 

241 An N x N matrix (N = number of amplifiers) that illustrates the 

242 connections between amplifiers within the detector layout. Each row 

243 and column index corresponds to the ampIds of a specific pair of 

244 amplifiers, and the matrix elements indicate their associations as 

245 follows: 

246 0: No association 

247 -1: Association exists (direction specified in the ampSides matrix) 

248 n >= 1: Diagonal elements indicate the number of neighboring 

249 amplifiers for the corresponding ampId==row==column number. 

250 

251 ampSides : `numpy.ndarray` 

252 An N x N matrix (N = the number of amplifiers) representing the amp 

253 side information corresponding to the `ampAssociations` 

254 matrix. The elements are integers defined as below: 

255 -1: No side due to no association or the same amp (diagonals) 

256 0: Side on the bottom 

257 1: Side on the right 

258 2: Side on the top 

259 3: Side on the left 

260 """ 

261 xCenters = [amp.getBBox().getCenterX() for amp in amps] 

262 yCenters = [amp.getBBox().getCenterY() for amp in amps] 

263 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1 

264 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1 

265 

266 nAmps = len(amps) 

267 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int) 

268 

269 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices): 

270 ampIds[yIndex, xIndex] = ampId 

271 

272 ampAssociations = np.zeros((nAmps, nAmps), dtype=int) 

273 ampSides = np.full_like(ampAssociations, -1) 

274 

275 for ampId in ampIds.ravel(): 

276 neighbors, sides = self.getNeighbors(ampIds, ampId) 

277 ampAssociations[ampId, neighbors] = -1 

278 ampSides[ampId, neighbors] = sides 

279 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum() 

280 

281 if ampAssociations.sum() != 0: 

282 raise RuntimeError("The `ampAssociations` array does not sum to zero.") 

283 

284 if not np.all(ampAssociations == ampAssociations.T): 

285 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.") 

286 

287 self.log.debug("amp associations:\n%s", ampAssociations) 

288 self.log.debug("amp sides:\n%s", ampSides) 

289 

290 return ampAssociations, ampSides 

291 

292 def getNeighbors(self, ampIds, ampId): 

293 """Get the neighbor amplifiers and their sides for a given 

294 amplifier. 

295 

296 Parameters 

297 ---------- 

298 ampIds : `numpy.ndarray` 

299 Matrix with amp side association information. 

300 ampId : `int` 

301 The amplifier ID for which neighbor amplifiers and side IDs 

302 are to be found. 

303 

304 Returns 

305 ------- 

306 neighbors : `list` [`int`] 

307 List of neighbor amplifier IDs. 

308 sides : `list` [`int`] 

309 List of side IDs, with each ID corresponding to its respective 

310 neighbor amplifier. 

311 """ 

312 m, n = ampIds.shape 

313 r, c = np.ravel(np.where(ampIds == ampId)) 

314 neighbors, sides = [], [] 

315 sideLookup = { 

316 0: (r + 1, c), 

317 1: (r, c + 1), 

318 2: (r - 1, c), 

319 3: (r, c - 1), 

320 } 

321 for side, (row, column) in sideLookup.items(): 

322 if 0 <= row < m and 0 <= column < n: 

323 neighbors.append(ampIds[row][column]) 

324 sides.append(side) 

325 return neighbors, sides 

326 

327 def getAmpOffsets(self, im, amps, associations, sides): 

328 """Calculate the amp offsets for all amplifiers. 

329 

330 Parameters 

331 ---------- 

332 im : `lsst.afw.image._image.ImageF` 

333 Amplifier image to extract data from. 

334 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

335 List of amplifier objects. 

336 associations : numpy.ndarray 

337 An N x N matrix containing amp association information, where N is 

338 the number of amplifiers. 

339 sides : numpy.ndarray 

340 An N x N matrix containing amp side information, where N is the 

341 number of amplifiers. 

342 

343 Returns 

344 ------- 

345 ampsOffsets : `numpy.ndarray` 

346 1D float array containing the calculated amp offsets for all 

347 amplifiers. 

348 """ 

349 ampsOffsets = np.zeros(len(amps)) 

350 ampsEdges = self.getAmpEdges(im, amps, sides) 

351 interfaceOffsetLookup = {} 

352 for ampId, ampAssociations in enumerate(associations): 

353 ampNeighbors = np.ravel(np.where(ampAssociations < 0)) 

354 for ampNeighbor in ampNeighbors: 

355 ampSide = sides[ampId][ampNeighbor] 

356 edgeA = ampsEdges[ampId][ampSide] 

357 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4] 

358 if ampId < ampNeighbor: 

359 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB) 

360 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset 

361 else: 

362 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"] 

363 ampsOffsets[ampId] += interfaceOffset 

364 return ampsOffsets 

365 

366 def getAmpEdges(self, im, amps, ampSides): 

367 """Calculate the amp edges for all amplifiers. 

368 

369 Parameters 

370 ---------- 

371 im : `lsst.afw.image._image.ImageF` 

372 Amplifier image to extract data from. 

373 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

374 List of amplifier objects. 

375 ampSides : `numpy.ndarray` 

376 An N x N matrix containing amp side information, where N is the 

377 number of amplifiers. 

378 

379 Returns 

380 ------- 

381 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]] 

382 A dictionary containing amp edge(s) for each amplifier, 

383 corresponding to one or more potential sides, where each edge is 

384 associated with a side. The outer dictionary has integer keys 

385 representing amplifier IDs, and the inner dictionary has integer 

386 keys representing side IDs for each amplifier and values that are 

387 1D arrays of floats representing the 1D medianified strips from the 

388 amp image, referred to as "amp edge": 

389 {ampID: {sideID: numpy.ndarray}, ...} 

390 """ 

391 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth 

392 ampEdges = {} 

393 slice_map = { 

394 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)), 

395 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)), 

396 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)), 

397 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)), 

398 } 

399 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)): 

400 ampEdges[ampId] = {} 

401 ampIm = im[amp.getBBox()].array 

402 # Loop over identified sides. 

403 for ampSide in ampSides: 

404 if ampSide < 0: 

405 continue 

406 strip = ampIm[slice_map[ampSide]] 

407 # Catch warnings to prevent all-NaN slice RuntimeWarning. 

408 with warnings.catch_warnings(): 

409 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered") 

410 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip 

411 return ampEdges 

412 

413 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB): 

414 """Calculate the amp offset for a given interface between two 

415 amplifiers. 

416 

417 Parameters 

418 ---------- 

419 ampIdA : int 

420 ID of the first amplifier. 

421 ampIdB : int 

422 ID of the second amplifier. 

423 edgeA : numpy.ndarray 

424 Amp edge for the first amplifier. 

425 edgeB : numpy.ndarray 

426 Amp edge for the second amplifier. 

427 

428 Returns 

429 ------- 

430 interfaceOffset : float 

431 The calculated amp offset value for the given interface between 

432 amps A and B. 

433 """ 

434 interfaceId = f"{ampIdA}{ampIdB}" 

435 sctrl = StatisticsControl() 

436 # NOTE: Taking the difference with the order below fixes the sign flip 

437 # in the B matrix. 

438 edgeDiff = edgeA - edgeB 

439 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff)) 

440 # Compute rolling averages. 

441 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same") 

442 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same") 

443 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None) 

444 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan 

445 # Take clipped mean of rolling average data as amp offset value. 

446 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue() 

447 # Perform a couple of do-no-harm safety checks: 

448 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac, 

449 # b) The absolute offset ADU value is < ampEdgeMaxOffset. 

450 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg)) 

451 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac 

452 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset 

453 if minFracFail or maxOffsetFail: 

454 interfaceOffset = 0 

455 self.log.warning( 

456 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold " 

457 f"({self.config.ampEdgeMinFrac}) or the absolute offset value exceeds the limit " 

458 f"({self.config.ampEdgeMaxOffset} ADU). Setting the interface offset to 0." 

459 ) 

460 self.log.debug( 

461 f"amp interface {interfaceId} : " 

462 f"viable edge difference frac = {ampEdgeGoodFrac}, " 

463 f"interface offset = {interfaceOffset:.3f}" 

464 ) 

465 return interfaceOffset