Coverage for python/lsst/ip/isr/ampOffset.py: 14%
155 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-12 10:46 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-12 10:46 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"]
24import warnings
26import numpy as np
27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics
28from lsst.afw.table import SourceTable
29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask
30from lsst.pex.config import Config, ConfigurableField, Field
31from lsst.pipe.base import Task, Struct
34class AmpOffsetConfig(Config):
35 """Configuration parameters for AmpOffsetTask."""
37 def setDefaults(self):
38 self.background.algorithm = "AKIMA_SPLINE"
39 self.background.useApprox = False
40 self.background.ignoredPixelMask = [
41 "BAD",
42 "SAT",
43 "INTRP",
44 "CR",
45 "EDGE",
46 "DETECTED",
47 "DETECTED_NEGATIVE",
48 "SUSPECT",
49 "NO_DATA",
50 ]
51 self.detection.reEstimateBackground = False
53 ampEdgeInset = Field(
54 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running "
55 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.",
56 dtype=int,
57 default=5,
58 )
59 ampEdgeWidth = Field(
60 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.",
61 dtype=int,
62 default=64,
63 )
64 ampEdgeMinFrac = Field(
65 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be "
66 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.",
67 dtype=float,
68 default=0.5,
69 )
70 ampEdgeMaxOffset = Field(
71 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the "
72 "result will be discarded and therefore not used to determine amp pedestal corrections.",
73 dtype=float,
74 default=5.0,
75 )
76 ampEdgeWindowFrac = Field(
77 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average "
78 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should "
79 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the "
80 "sliding window used in obs_subaru for compatibility with existing HSC data.",
81 dtype=float,
82 default=512 / 4176,
83 )
84 doBackground = Field(
85 doc="Estimate and subtract background prior to amp offset estimation?",
86 dtype=bool,
87 default=True,
88 )
89 background = ConfigurableField(
90 doc="An initial background estimation step run prior to amp offset calculation.",
91 target=SubtractBackgroundTask,
92 )
93 backgroundFractionSample = Field(
94 doc="The fraction of the shorter side of the amplifier used for background binning.",
95 dtype=float,
96 default=1.0,
97 )
98 doDetection = Field(
99 doc="Detect sources and update cloned exposure prior to amp offset estimation?",
100 dtype=bool,
101 default=True,
102 )
103 detection = ConfigurableField(
104 doc="Source detection to add temporary detection footprints prior to amp offset calculation.",
105 target=SourceDetectionTask,
106 )
109class AmpOffsetTask(Task):
110 """Calculate and apply amp offset corrections to an exposure."""
112 ConfigClass = AmpOffsetConfig
113 _DefaultName = "isrAmpOffset"
115 def __init__(self, *args, **kwargs):
116 super().__init__(*args, **kwargs)
117 # Always load background subtask, even if doBackground=False;
118 # this allows for default plane bit masks to be defined.
119 self.makeSubtask("background")
120 if self.config.doDetection:
121 self.makeSubtask("detection")
122 # Initialize all of the instance variables here.
123 self.shortAmpSide = 0
125 def run(self, exposure):
126 """Calculate amp offset values, determine corrective pedestals for each
127 amp, and update the input exposure in-place.
129 Parameters
130 ----------
131 exposure: `lsst.afw.image.Exposure`
132 Exposure to be corrected for amp offsets.
133 """
135 # Generate an exposure clone to work on and establish the bit mask.
136 exp = exposure.clone()
137 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask)
138 amps = exp.getDetector().getAmplifiers()
140 # Check that all amps have the same gemotry.
141 ampDims = [amp.getBBox().getDimensions() for amp in amps]
142 if not all(dim == ampDims[0] for dim in ampDims):
143 raise RuntimeError("All amps should have the same geometry.")
145 # Assuming all the amps have the same geometry.
146 self.shortAmpSide = np.min(ampDims[0])
148 # Fit and subtract background.
149 if self.config.doBackground:
150 maskedImage = exp.getMaskedImage()
151 # Assuming all the detectors are the same.
152 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
153 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
154 # This ensures that the `binSize` is as large as possible,
155 # preventing background subtraction from inadvertently removing the
156 # amp offset signature. Here it's set to the shorter dimension of
157 # the amplifier by default (`backgroundFractionSample` = 1), which
158 # seems reasonable.
159 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY))
160 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
161 maskedImage -= bgImage
163 # Detect sources and update cloned exposure mask planes in-place.
164 if self.config.doDetection:
165 schema = SourceTable.makeMinimalSchema()
166 table = SourceTable.make(schema)
167 # Detection sigma, used for smoothing and to grow detections, is
168 # normally measured from the PSF of the exposure. As the PSF hasn't
169 # been measured at this stage of processing, sigma is instead
170 # set to an approximate value here (which should be sufficient).
171 _ = self.detection.run(table=table, exposure=exp, sigma=2)
173 # Safety check: do any pixels remain for amp offset estimation?
174 if (exp.mask.array & bitMask).all():
175 self.log.warning(
176 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set "
177 "to zero."
178 )
179 pedestals = np.zeros(len(amps))
180 else:
181 # Set up amp offset inputs.
182 im = exp.image
183 im.array[(exp.mask.array & bitMask) > 0] = np.nan
185 if self.config.ampEdgeWindowFrac > 1:
186 raise RuntimeError(
187 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the "
188 "edge length exceeds 1. This leads to complications downstream, after convolution in "
189 "the `getSideAmpOffset()` method. Please modify the `ampEdgeWindowFrac` value in the "
190 "config to be 1 or less and rerun."
191 )
193 # Determine amplifier geometry.
194 ampAreas = {amp.getBBox().getArea() for amp in amps}
195 if len(ampAreas) > 1:
196 raise NotImplementedError(
197 "Amp offset correction is not yet implemented for detectors with differing amp sizes."
198 )
200 # Obtain association and offset matrices.
201 A, sides = self.getAmpAssociations(amps)
202 B = self.getAmpOffsets(im, amps, A, sides)
204 # If least-squares minimization fails, convert NaNs to zeroes,
205 # ensuring that no values are erroneously added/subtracted.
206 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0])
208 metadata = exposure.getMetadata()
209 for amp, pedestal in zip(amps, pedestals):
210 ampIm = exposure.image[amp.getBBox()].array
211 ampIm -= pedestal
212 ampName = amp.getName()
213 metadata.set(
214 f"LSST ISR AMPOFFSET PEDESTAL {ampName}",
215 float(pedestal),
216 f"Pedestal level subtracted from amp {ampName}",
217 )
218 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}")
220 return Struct(pedestals=pedestals)
222 def getAmpAssociations(self, amps):
223 """Determine amp geometry and amp associations from a list of
224 amplifiers.
226 Parse an input list of amplifiers to determine the layout of amps
227 within a detector, and identify all amp sides (i.e., the
228 horizontal and vertical junctions between amps).
230 Returns a matrix with a shape corresponding to the geometry of the amps
231 in the detector.
233 Parameters
234 ----------
235 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
236 List of amplifier objects used to deduce associations.
238 Returns
239 -------
240 ampAssociations : `numpy.ndarray`
241 An N x N matrix (N = number of amplifiers) that illustrates the
242 connections between amplifiers within the detector layout. Each row
243 and column index corresponds to the ampIds of a specific pair of
244 amplifiers, and the matrix elements indicate their associations as
245 follows:
246 0: No association
247 -1: Association exists (direction specified in the ampSides matrix)
248 n >= 1: Diagonal elements indicate the number of neighboring
249 amplifiers for the corresponding ampId==row==column number.
251 ampSides : `numpy.ndarray`
252 An N x N matrix (N = the number of amplifiers) representing the amp
253 side information corresponding to the `ampAssociations`
254 matrix. The elements are integers defined as below:
255 -1: No side due to no association or the same amp (diagonals)
256 0: Side on the bottom
257 1: Side on the right
258 2: Side on the top
259 3: Side on the left
260 """
261 xCenters = [amp.getBBox().getCenterX() for amp in amps]
262 yCenters = [amp.getBBox().getCenterY() for amp in amps]
263 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1
264 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1
266 nAmps = len(amps)
267 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int)
269 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices):
270 ampIds[yIndex, xIndex] = ampId
272 ampAssociations = np.zeros((nAmps, nAmps), dtype=int)
273 ampSides = np.full_like(ampAssociations, -1)
275 for ampId in ampIds.ravel():
276 neighbors, sides = self.getNeighbors(ampIds, ampId)
277 ampAssociations[ampId, neighbors] = -1
278 ampSides[ampId, neighbors] = sides
279 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum()
281 if ampAssociations.sum() != 0:
282 raise RuntimeError("The `ampAssociations` array does not sum to zero.")
284 if not np.all(ampAssociations == ampAssociations.T):
285 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.")
287 self.log.debug("amp associations:\n%s", ampAssociations)
288 self.log.debug("amp sides:\n%s", ampSides)
290 return ampAssociations, ampSides
292 def getNeighbors(self, ampIds, ampId):
293 """Get the neighbor amplifiers and their sides for a given
294 amplifier.
296 Parameters
297 ----------
298 ampIds : `numpy.ndarray`
299 Matrix with amp side association information.
300 ampId : `int`
301 The amplifier ID for which neighbor amplifiers and side IDs
302 are to be found.
304 Returns
305 -------
306 neighbors : `list` [`int`]
307 List of neighbor amplifier IDs.
308 sides : `list` [`int`]
309 List of side IDs, with each ID corresponding to its respective
310 neighbor amplifier.
311 """
312 m, n = ampIds.shape
313 r, c = np.ravel(np.where(ampIds == ampId))
314 neighbors, sides = [], []
315 sideLookup = {
316 0: (r + 1, c),
317 1: (r, c + 1),
318 2: (r - 1, c),
319 3: (r, c - 1),
320 }
321 for side, (row, column) in sideLookup.items():
322 if 0 <= row < m and 0 <= column < n:
323 neighbors.append(ampIds[row][column])
324 sides.append(side)
325 return neighbors, sides
327 def getAmpOffsets(self, im, amps, associations, sides):
328 """Calculate the amp offsets for all amplifiers.
330 Parameters
331 ----------
332 im : `lsst.afw.image._image.ImageF`
333 Amplifier image to extract data from.
334 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
335 List of amplifier objects.
336 associations : numpy.ndarray
337 An N x N matrix containing amp association information, where N is
338 the number of amplifiers.
339 sides : numpy.ndarray
340 An N x N matrix containing amp side information, where N is the
341 number of amplifiers.
343 Returns
344 -------
345 ampsOffsets : `numpy.ndarray`
346 1D float array containing the calculated amp offsets for all
347 amplifiers.
348 """
349 ampsOffsets = np.zeros(len(amps))
350 ampsEdges = self.getAmpEdges(im, amps, sides)
351 interfaceOffsetLookup = {}
352 for ampId, ampAssociations in enumerate(associations):
353 ampNeighbors = np.ravel(np.where(ampAssociations < 0))
354 for ampNeighbor in ampNeighbors:
355 ampSide = sides[ampId][ampNeighbor]
356 edgeA = ampsEdges[ampId][ampSide]
357 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4]
358 if ampId < ampNeighbor:
359 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB)
360 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset
361 else:
362 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"]
363 ampsOffsets[ampId] += interfaceOffset
364 return ampsOffsets
366 def getAmpEdges(self, im, amps, ampSides):
367 """Calculate the amp edges for all amplifiers.
369 Parameters
370 ----------
371 im : `lsst.afw.image._image.ImageF`
372 Amplifier image to extract data from.
373 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
374 List of amplifier objects.
375 ampSides : `numpy.ndarray`
376 An N x N matrix containing amp side information, where N is the
377 number of amplifiers.
379 Returns
380 -------
381 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]]
382 A dictionary containing amp edge(s) for each amplifier,
383 corresponding to one or more potential sides, where each edge is
384 associated with a side. The outer dictionary has integer keys
385 representing amplifier IDs, and the inner dictionary has integer
386 keys representing side IDs for each amplifier and values that are
387 1D arrays of floats representing the 1D medianified strips from the
388 amp image, referred to as "amp edge":
389 {ampID: {sideID: numpy.ndarray}, ...}
390 """
391 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth
392 ampEdges = {}
393 slice_map = {
394 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)),
395 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)),
396 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)),
397 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)),
398 }
399 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)):
400 ampEdges[ampId] = {}
401 ampIm = im[amp.getBBox()].array
402 # Loop over identified sides.
403 for ampSide in ampSides:
404 if ampSide < 0:
405 continue
406 strip = ampIm[slice_map[ampSide]]
407 # Catch warnings to prevent all-NaN slice RuntimeWarning.
408 with warnings.catch_warnings():
409 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered")
410 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip
411 return ampEdges
413 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB):
414 """Calculate the amp offset for a given interface between two
415 amplifiers.
417 Parameters
418 ----------
419 ampIdA : int
420 ID of the first amplifier.
421 ampIdB : int
422 ID of the second amplifier.
423 edgeA : numpy.ndarray
424 Amp edge for the first amplifier.
425 edgeB : numpy.ndarray
426 Amp edge for the second amplifier.
428 Returns
429 -------
430 interfaceOffset : float
431 The calculated amp offset value for the given interface between
432 amps A and B.
433 """
434 interfaceId = f"{ampIdA}{ampIdB}"
435 sctrl = StatisticsControl()
436 # NOTE: Taking the difference with the order below fixes the sign flip
437 # in the B matrix.
438 edgeDiff = edgeA - edgeB
439 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff))
440 # Compute rolling averages.
441 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same")
442 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same")
443 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None)
444 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan
445 # Take clipped mean of rolling average data as amp offset value.
446 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue()
447 # Perform a couple of do-no-harm safety checks:
448 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac,
449 # b) The absolute offset ADU value is < ampEdgeMaxOffset.
450 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg))
451 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac
452 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset
453 if minFracFail or maxOffsetFail:
454 interfaceOffset = 0
455 self.log.warning(
456 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold "
457 f"({self.config.ampEdgeMinFrac}) or the absolute offset value exceeds the limit "
458 f"({self.config.ampEdgeMaxOffset} ADU). Setting the interface offset to 0."
459 )
460 self.log.debug(
461 f"amp interface {interfaceId} : "
462 f"viable edge difference frac = {ampEdgeGoodFrac}, "
463 f"interface offset = {interfaceOffset:.3f}"
464 )
465 return interfaceOffset