Coverage for python/lsst/ip/isr/ampOffset.py: 13%

167 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-02-27 13:25 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"] 

23 

24import warnings 

25 

26import numpy as np 

27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics 

28from lsst.afw.table import SourceTable 

29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask 

30from lsst.pex.config import Config, ConfigurableField, Field 

31from lsst.pipe.base import Struct, Task 

32 

33 

34class AmpOffsetConfig(Config): 

35 """Configuration parameters for AmpOffsetTask.""" 

36 

37 def setDefaults(self): 

38 self.background.algorithm = "AKIMA_SPLINE" 

39 self.background.useApprox = False 

40 self.background.ignoredPixelMask = [ 

41 "BAD", 

42 "SAT", 

43 "INTRP", 

44 "CR", 

45 "EDGE", 

46 "DETECTED", 

47 "DETECTED_NEGATIVE", 

48 "SUSPECT", 

49 "NO_DATA", 

50 ] 

51 self.detection.reEstimateBackground = False 

52 

53 # This maintains existing behavior and test values after DM-39796. 

54 self.detection.thresholdType = "stdev" 

55 

56 ampEdgeInset = Field( 

57 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running " 

58 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.", 

59 dtype=int, 

60 default=5, 

61 ) 

62 ampEdgeWidth = Field( 

63 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.", 

64 dtype=int, 

65 default=64, 

66 ) 

67 ampEdgeMinFrac = Field( 

68 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be " 

69 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.", 

70 dtype=float, 

71 default=0.5, 

72 ) 

73 ampEdgeMaxOffset = Field( 

74 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the " 

75 "result will be discarded and therefore not used to determine amp pedestal corrections.", 

76 dtype=float, 

77 default=5.0, 

78 ) 

79 ampEdgeWindowFrac = Field( 

80 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average " 

81 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should " 

82 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the " 

83 "sliding window used in obs_subaru for compatibility with existing HSC data.", 

84 dtype=float, 

85 default=512 / 4176, 

86 ) 

87 doBackground = Field( 

88 doc="Estimate and subtract background prior to amp offset estimation?", 

89 dtype=bool, 

90 default=True, 

91 ) 

92 background = ConfigurableField( 

93 doc="An initial background estimation step run prior to amp offset calculation.", 

94 target=SubtractBackgroundTask, 

95 ) 

96 backgroundFractionSample = Field( 

97 doc="The fraction of the shorter side of the amplifier used for background binning.", 

98 dtype=float, 

99 default=1.0, 

100 ) 

101 doDetection = Field( 

102 doc="Detect sources and update cloned exposure prior to amp offset estimation?", 

103 dtype=bool, 

104 default=True, 

105 ) 

106 detection = ConfigurableField( 

107 doc="Source detection to add temporary detection footprints prior to amp offset calculation.", 

108 target=SourceDetectionTask, 

109 ) 

110 applyWeights = Field( 

111 doc="Weights the amp offset calculation by the length of the interface between amplifiers. Applying " 

112 "weights does not affect outcomes for amplifiers in a 2D grid with square-shaped amplifiers or in " 

113 "any 1D layout on a detector, regardless of whether the amplifiers are square.", 

114 dtype=bool, 

115 default=True, 

116 ) 

117 

118 

119class AmpOffsetTask(Task): 

120 """Calculate and apply amp offset corrections to an exposure.""" 

121 

122 ConfigClass = AmpOffsetConfig 

123 _DefaultName = "isrAmpOffset" 

124 

125 def __init__(self, *args, **kwargs): 

126 super().__init__(*args, **kwargs) 

127 # Always load background subtask, even if doBackground=False; 

128 # this allows for default plane bit masks to be defined. 

129 self.makeSubtask("background") 

130 if self.config.doDetection: 

131 self.makeSubtask("detection") 

132 # Initialize all of the instance variables here. 

133 self.shortAmpSide = 0 

134 

135 def run(self, exposure): 

136 """Calculate amp offset values, determine corrective pedestals for each 

137 amp, and update the input exposure in-place. 

138 

139 Parameters 

140 ---------- 

141 exposure: `lsst.afw.image.Exposure` 

142 Exposure to be corrected for amp offsets. 

143 """ 

144 

145 # Generate an exposure clone to work on and establish the bit mask. 

146 exp = exposure.clone() 

147 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask) 

148 amps = exp.getDetector().getAmplifiers() 

149 

150 # Check that all amps have the same gemotry. 

151 ampDims = [amp.getBBox().getDimensions() for amp in amps] 

152 if not all(dim == ampDims[0] for dim in ampDims): 

153 raise RuntimeError("All amps should have the same geometry.") 

154 else: 

155 # The zeroth amp is representative of all amps in the detector. 

156 self.ampDims = ampDims[0] 

157 # Dictionary mapping side numbers to interface lengths. 

158 # See `getAmpAssociations()` for details about sides. 

159 self.interfaceLengthLookupBySide = {i: self.ampDims[i % 2] for i in range(4)} 

160 

161 # Determine amplifier geometry. 

162 ampWidths = {amp.getBBox().getWidth() for amp in amps} 

163 ampHeights = {amp.getBBox().getHeight() for amp in amps} 

164 if len(ampWidths) > 1 or len(ampHeights) > 1: 

165 raise NotImplementedError( 

166 "Amp offset correction is not yet implemented for detectors with differing amp sizes." 

167 ) 

168 

169 # Assuming all the amps have the same geometry. 

170 self.shortAmpSide = np.min(ampDims[0]) 

171 

172 # Check that the edge width and inset are not too large. 

173 if self.config.ampEdgeWidth >= self.shortAmpSide - 2 * self.config.ampEdgeInset: 

174 raise RuntimeError( 

175 f"The edge width ({self.config.ampEdgeWidth}) plus insets ({self.config.ampEdgeInset}) " 

176 f"exceed the amp's short side ({self.shortAmpSide}). This setup leads to incorrect results." 

177 ) 

178 

179 # Fit and subtract background. 

180 if self.config.doBackground: 

181 maskedImage = exp.getMaskedImage() 

182 # Assuming all the detectors are the same. 

183 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1 

184 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1 

185 # This ensures that the `binSize` is as large as possible, 

186 # preventing background subtraction from inadvertently removing the 

187 # amp offset signature. Here it's set to the shorter dimension of 

188 # the amplifier by default (`backgroundFractionSample` = 1), which 

189 # seems reasonable. 

190 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY)) 

191 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle) 

192 maskedImage -= bgImage 

193 

194 # Detect sources and update cloned exposure mask planes in-place. 

195 if self.config.doDetection: 

196 schema = SourceTable.makeMinimalSchema() 

197 table = SourceTable.make(schema) 

198 # Detection sigma, used for smoothing and to grow detections, is 

199 # normally measured from the PSF of the exposure. As the PSF hasn't 

200 # been measured at this stage of processing, sigma is instead 

201 # set to an approximate value here (which should be sufficient). 

202 _ = self.detection.run(table=table, exposure=exp, sigma=2) 

203 

204 # Safety check: do any pixels remain for amp offset estimation? 

205 if (exp.mask.array & bitMask).all(): 

206 self.log.warning( 

207 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set " 

208 "to zero." 

209 ) 

210 pedestals = np.zeros(len(amps)) 

211 else: 

212 # Set up amp offset inputs. 

213 im = exp.image 

214 im.array[(exp.mask.array & bitMask) > 0] = np.nan 

215 

216 if self.config.ampEdgeWindowFrac > 1: 

217 raise RuntimeError( 

218 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the " 

219 "edge length exceeds 1. This leads to complications downstream, after convolution in " 

220 "the `getInterfaceOffset()` method. Please modify the `ampEdgeWindowFrac` value in the " 

221 "config to be 1 or less and rerun." 

222 ) 

223 

224 # Obtain association and offset matrices. 

225 A, sides = self.getAmpAssociations(amps) 

226 B = self.getAmpOffsets(im, amps, A, sides) 

227 

228 # If least-squares minimization fails, convert NaNs to zeroes, 

229 # ensuring that no values are erroneously added/subtracted. 

230 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0]) 

231 

232 metadata = exposure.getMetadata() 

233 for amp, pedestal in zip(amps, pedestals): 

234 ampIm = exposure.image[amp.getBBox()].array 

235 ampIm -= pedestal 

236 ampName = amp.getName() 

237 metadata.set( 

238 f"LSST ISR AMPOFFSET PEDESTAL {ampName}", 

239 float(pedestal), 

240 f"Pedestal level subtracted from amp {ampName}", 

241 ) 

242 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}") 

243 

244 return Struct(pedestals=pedestals) 

245 

246 def getAmpAssociations(self, amps): 

247 """Determine amp geometry and amp associations from a list of 

248 amplifiers. 

249 

250 Parse an input list of amplifiers to determine the layout of amps 

251 within a detector, and identify all amp sides (i.e., the 

252 horizontal and vertical junctions between amps). 

253 

254 Returns a matrix with a shape corresponding to the geometry of the amps 

255 in the detector. 

256 

257 Parameters 

258 ---------- 

259 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

260 List of amplifier objects used to deduce associations. 

261 

262 Returns 

263 ------- 

264 ampAssociations : `numpy.ndarray` 

265 An N x N matrix (N = number of amplifiers) that illustrates the 

266 connections between amplifiers within the detector layout. Each row 

267 and column index corresponds to the ampIds of a specific pair of 

268 amplifiers, and the matrix elements indicate their associations as 

269 follows: 

270 0: No association 

271 -1: Association exists (direction specified in the ampSides matrix) 

272 n >= 1: Diagonal elements indicate the number of neighboring 

273 amplifiers for the corresponding ampId==row==column number. 

274 

275 ampSides : `numpy.ndarray` 

276 An N x N matrix (N = the number of amplifiers) representing the amp 

277 side information corresponding to the `ampAssociations` 

278 matrix. The elements are integers defined as below: 

279 -1: No side due to no association or the same amp (diagonals) 

280 0: Side on the bottom 

281 1: Side on the right 

282 2: Side on the top 

283 3: Side on the left 

284 """ 

285 xCenters = [amp.getBBox().getCenterX() for amp in amps] 

286 yCenters = [amp.getBBox().getCenterY() for amp in amps] 

287 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1 

288 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1 

289 

290 nAmps = len(amps) 

291 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int) 

292 

293 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices): 

294 ampIds[yIndex, xIndex] = ampId 

295 

296 ampAssociations = np.zeros((nAmps, nAmps), dtype=int) 

297 ampSides = np.full_like(ampAssociations, -1) 

298 

299 for ampId in ampIds.ravel(): 

300 neighbors, sides = self.getNeighbors(ampIds, ampId) 

301 interfaceWeights = ( 

302 1 

303 if not self.config.applyWeights 

304 else np.array([self.interfaceLengthLookupBySide[side] for side in sides]) 

305 ) 

306 ampAssociations[ampId, neighbors] = -1 * interfaceWeights 

307 ampSides[ampId, neighbors] = sides 

308 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum() 

309 

310 if ampAssociations.sum() != 0: 

311 raise RuntimeError("The `ampAssociations` array does not sum to zero.") 

312 

313 if not np.all(ampAssociations == ampAssociations.T): 

314 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.") 

315 

316 self.log.debug("amp associations:\n%s", ampAssociations) 

317 self.log.debug("amp sides:\n%s", ampSides) 

318 

319 return ampAssociations, ampSides 

320 

321 def getNeighbors(self, ampIds, ampId): 

322 """Get the neighbor amplifiers and their sides for a given 

323 amplifier. 

324 

325 Parameters 

326 ---------- 

327 ampIds : `numpy.ndarray` 

328 Matrix with amp side association information. 

329 ampId : `int` 

330 The amplifier ID for which neighbor amplifiers and side IDs 

331 are to be found. 

332 

333 Returns 

334 ------- 

335 neighbors : `list` [`int`] 

336 List of neighbor amplifier IDs. 

337 sides : `list` [`int`] 

338 List of side IDs, with each ID corresponding to its respective 

339 neighbor amplifier. 

340 """ 

341 m, n = ampIds.shape 

342 r, c = np.ravel(np.where(ampIds == ampId)) 

343 neighbors, sides = [], [] 

344 sideLookup = { 

345 0: (r + 1, c), 

346 1: (r, c + 1), 

347 2: (r - 1, c), 

348 3: (r, c - 1), 

349 } 

350 for side, (row, column) in sideLookup.items(): 

351 if 0 <= row < m and 0 <= column < n: 

352 neighbors.append(ampIds[row][column]) 

353 sides.append(side) 

354 return neighbors, sides 

355 

356 def getAmpOffsets(self, im, amps, associations, sides): 

357 """Calculate the amp offsets for all amplifiers. 

358 

359 Parameters 

360 ---------- 

361 im : `lsst.afw.image._image.ImageF` 

362 Amplifier image to extract data from. 

363 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

364 List of amplifier objects. 

365 associations : numpy.ndarray 

366 An N x N matrix containing amp association information, where N is 

367 the number of amplifiers. 

368 sides : numpy.ndarray 

369 An N x N matrix containing amp side information, where N is the 

370 number of amplifiers. 

371 

372 Returns 

373 ------- 

374 ampsOffsets : `numpy.ndarray` 

375 1D float array containing the calculated amp offsets for all 

376 amplifiers. 

377 """ 

378 ampsOffsets = np.zeros(len(amps)) 

379 ampsEdges = self.getAmpEdges(im, amps, sides) 

380 interfaceOffsetLookup = {} 

381 

382 for ampId, ampAssociations in enumerate(associations): 

383 ampNeighbors = np.ravel(np.where(ampAssociations < 0)) 

384 for ampNeighbor in ampNeighbors: 

385 ampSide = sides[ampId][ampNeighbor] 

386 interfaceWeight = ( 

387 1 if not self.config.applyWeights else self.interfaceLengthLookupBySide[ampSide] 

388 ) 

389 edgeA = ampsEdges[ampId][ampSide] 

390 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4] 

391 if ampId < ampNeighbor: 

392 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB) 

393 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset 

394 else: 

395 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"] 

396 ampsOffsets[ampId] += interfaceWeight * interfaceOffset 

397 return ampsOffsets 

398 

399 def getAmpEdges(self, im, amps, ampSides): 

400 """Calculate the amp edges for all amplifiers. 

401 

402 Parameters 

403 ---------- 

404 im : `lsst.afw.image._image.ImageF` 

405 Amplifier image to extract data from. 

406 amps : `list` [`lsst.afw.cameraGeom.Amplifier`] 

407 List of amplifier objects. 

408 ampSides : `numpy.ndarray` 

409 An N x N matrix containing amp side information, where N is the 

410 number of amplifiers. 

411 

412 Returns 

413 ------- 

414 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]] 

415 A dictionary containing amp edge(s) for each amplifier, 

416 corresponding to one or more potential sides, where each edge is 

417 associated with a side. The outer dictionary has integer keys 

418 representing amplifier IDs, and the inner dictionary has integer 

419 keys representing side IDs for each amplifier and values that are 

420 1D arrays of floats representing the 1D medianified strips from the 

421 amp image, referred to as "amp edge": 

422 {ampID: {sideID: numpy.ndarray}, ...} 

423 """ 

424 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth 

425 ampEdges = {} 

426 slice_map = { 

427 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)), 

428 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)), 

429 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)), 

430 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)), 

431 } 

432 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)): 

433 ampEdges[ampId] = {} 

434 ampIm = im[amp.getBBox()].array 

435 # Loop over identified sides. 

436 for ampSide in ampSides: 

437 if ampSide < 0: 

438 continue 

439 strip = ampIm[slice_map[ampSide]] 

440 # Catch warnings to prevent all-NaN slice RuntimeWarning. 

441 with warnings.catch_warnings(): 

442 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered") 

443 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip 

444 return ampEdges 

445 

446 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB): 

447 """Calculate the amp offset for a given interface between two 

448 amplifiers. 

449 

450 Parameters 

451 ---------- 

452 ampIdA : int 

453 ID of the first amplifier. 

454 ampIdB : int 

455 ID of the second amplifier. 

456 edgeA : numpy.ndarray 

457 Amp edge for the first amplifier. 

458 edgeB : numpy.ndarray 

459 Amp edge for the second amplifier. 

460 

461 Returns 

462 ------- 

463 interfaceOffset : float 

464 The calculated amp offset value for the given interface between 

465 amps A and B. 

466 """ 

467 interfaceId = f"{ampIdA}{ampIdB}" 

468 sctrl = StatisticsControl() 

469 # NOTE: Taking the difference with the order below fixes the sign flip 

470 # in the B matrix. 

471 edgeDiff = edgeA - edgeB 

472 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff)) 

473 # Compute rolling averages. 

474 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same") 

475 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same") 

476 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None) 

477 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan 

478 # Take clipped mean of rolling average data as amp offset value. 

479 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue() 

480 # Perform a couple of do-no-harm safety checks: 

481 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac, 

482 # b) The absolute offset ADU value is < ampEdgeMaxOffset. 

483 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg)) 

484 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac 

485 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset 

486 if minFracFail or maxOffsetFail: 

487 interfaceOffset = 0 

488 if minFracFail: 

489 self.log.warning( 

490 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold " 

491 f"({ampEdgeGoodFrac:.2f} < {self.config.ampEdgeMinFrac}). Setting the interface offset " 

492 f"to {interfaceOffset}." 

493 ) 

494 if maxOffsetFail: 

495 self.log.warning( 

496 "The absolute offset value exceeds the limit " 

497 f"({np.abs(interfaceOffset):.2f} > {self.config.ampEdgeMaxOffset} ADU). Setting the " 

498 f"interface offset to {interfaceOffset}." 

499 ) 

500 self.log.debug( 

501 f"amp interface {interfaceId} : " 

502 f"viable edge difference frac = {ampEdgeGoodFrac}, " 

503 f"interface offset = {interfaceOffset:.3f}" 

504 ) 

505 return interfaceOffset