Coverage for python/lsst/ip/isr/ampOffset.py: 14%
156 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-20 10:19 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-20 10:19 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"]
24import warnings
26import numpy as np
27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics
28from lsst.afw.table import SourceTable
29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask
30from lsst.pex.config import Config, ConfigurableField, Field
31from lsst.pipe.base import Task, Struct
34class AmpOffsetConfig(Config):
35 """Configuration parameters for AmpOffsetTask."""
37 def setDefaults(self):
38 self.background.algorithm = "AKIMA_SPLINE"
39 self.background.useApprox = False
40 self.background.ignoredPixelMask = [
41 "BAD",
42 "SAT",
43 "INTRP",
44 "CR",
45 "EDGE",
46 "DETECTED",
47 "DETECTED_NEGATIVE",
48 "SUSPECT",
49 "NO_DATA",
50 ]
51 self.detection.reEstimateBackground = False
53 # This maintains existing behavior and test values after DM-39796.
54 self.detection.thresholdType = "stdev"
56 ampEdgeInset = Field(
57 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running "
58 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.",
59 dtype=int,
60 default=5,
61 )
62 ampEdgeWidth = Field(
63 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.",
64 dtype=int,
65 default=64,
66 )
67 ampEdgeMinFrac = Field(
68 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be "
69 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.",
70 dtype=float,
71 default=0.5,
72 )
73 ampEdgeMaxOffset = Field(
74 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the "
75 "result will be discarded and therefore not used to determine amp pedestal corrections.",
76 dtype=float,
77 default=5.0,
78 )
79 ampEdgeWindowFrac = Field(
80 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average "
81 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should "
82 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the "
83 "sliding window used in obs_subaru for compatibility with existing HSC data.",
84 dtype=float,
85 default=512 / 4176,
86 )
87 doBackground = Field(
88 doc="Estimate and subtract background prior to amp offset estimation?",
89 dtype=bool,
90 default=True,
91 )
92 background = ConfigurableField(
93 doc="An initial background estimation step run prior to amp offset calculation.",
94 target=SubtractBackgroundTask,
95 )
96 backgroundFractionSample = Field(
97 doc="The fraction of the shorter side of the amplifier used for background binning.",
98 dtype=float,
99 default=1.0,
100 )
101 doDetection = Field(
102 doc="Detect sources and update cloned exposure prior to amp offset estimation?",
103 dtype=bool,
104 default=True,
105 )
106 detection = ConfigurableField(
107 doc="Source detection to add temporary detection footprints prior to amp offset calculation.",
108 target=SourceDetectionTask,
109 )
112class AmpOffsetTask(Task):
113 """Calculate and apply amp offset corrections to an exposure."""
115 ConfigClass = AmpOffsetConfig
116 _DefaultName = "isrAmpOffset"
118 def __init__(self, *args, **kwargs):
119 super().__init__(*args, **kwargs)
120 # Always load background subtask, even if doBackground=False;
121 # this allows for default plane bit masks to be defined.
122 self.makeSubtask("background")
123 if self.config.doDetection:
124 self.makeSubtask("detection")
125 # Initialize all of the instance variables here.
126 self.shortAmpSide = 0
128 def run(self, exposure):
129 """Calculate amp offset values, determine corrective pedestals for each
130 amp, and update the input exposure in-place.
132 Parameters
133 ----------
134 exposure: `lsst.afw.image.Exposure`
135 Exposure to be corrected for amp offsets.
136 """
138 # Generate an exposure clone to work on and establish the bit mask.
139 exp = exposure.clone()
140 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask)
141 amps = exp.getDetector().getAmplifiers()
143 # Check that all amps have the same gemotry.
144 ampDims = [amp.getBBox().getDimensions() for amp in amps]
145 if not all(dim == ampDims[0] for dim in ampDims):
146 raise RuntimeError("All amps should have the same geometry.")
148 # Assuming all the amps have the same geometry.
149 self.shortAmpSide = np.min(ampDims[0])
151 # Fit and subtract background.
152 if self.config.doBackground:
153 maskedImage = exp.getMaskedImage()
154 # Assuming all the detectors are the same.
155 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
156 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
157 # This ensures that the `binSize` is as large as possible,
158 # preventing background subtraction from inadvertently removing the
159 # amp offset signature. Here it's set to the shorter dimension of
160 # the amplifier by default (`backgroundFractionSample` = 1), which
161 # seems reasonable.
162 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY))
163 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
164 maskedImage -= bgImage
166 # Detect sources and update cloned exposure mask planes in-place.
167 if self.config.doDetection:
168 schema = SourceTable.makeMinimalSchema()
169 table = SourceTable.make(schema)
170 # Detection sigma, used for smoothing and to grow detections, is
171 # normally measured from the PSF of the exposure. As the PSF hasn't
172 # been measured at this stage of processing, sigma is instead
173 # set to an approximate value here (which should be sufficient).
174 _ = self.detection.run(table=table, exposure=exp, sigma=2)
176 # Safety check: do any pixels remain for amp offset estimation?
177 if (exp.mask.array & bitMask).all():
178 self.log.warning(
179 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set "
180 "to zero."
181 )
182 pedestals = np.zeros(len(amps))
183 else:
184 # Set up amp offset inputs.
185 im = exp.image
186 im.array[(exp.mask.array & bitMask) > 0] = np.nan
188 if self.config.ampEdgeWindowFrac > 1:
189 raise RuntimeError(
190 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the "
191 "edge length exceeds 1. This leads to complications downstream, after convolution in "
192 "the `getSideAmpOffset()` method. Please modify the `ampEdgeWindowFrac` value in the "
193 "config to be 1 or less and rerun."
194 )
196 # Determine amplifier geometry.
197 ampAreas = {amp.getBBox().getArea() for amp in amps}
198 if len(ampAreas) > 1:
199 raise NotImplementedError(
200 "Amp offset correction is not yet implemented for detectors with differing amp sizes."
201 )
203 # Obtain association and offset matrices.
204 A, sides = self.getAmpAssociations(amps)
205 B = self.getAmpOffsets(im, amps, A, sides)
207 # If least-squares minimization fails, convert NaNs to zeroes,
208 # ensuring that no values are erroneously added/subtracted.
209 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0])
211 metadata = exposure.getMetadata()
212 for amp, pedestal in zip(amps, pedestals):
213 ampIm = exposure.image[amp.getBBox()].array
214 ampIm -= pedestal
215 ampName = amp.getName()
216 metadata.set(
217 f"LSST ISR AMPOFFSET PEDESTAL {ampName}",
218 float(pedestal),
219 f"Pedestal level subtracted from amp {ampName}",
220 )
221 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}")
223 return Struct(pedestals=pedestals)
225 def getAmpAssociations(self, amps):
226 """Determine amp geometry and amp associations from a list of
227 amplifiers.
229 Parse an input list of amplifiers to determine the layout of amps
230 within a detector, and identify all amp sides (i.e., the
231 horizontal and vertical junctions between amps).
233 Returns a matrix with a shape corresponding to the geometry of the amps
234 in the detector.
236 Parameters
237 ----------
238 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
239 List of amplifier objects used to deduce associations.
241 Returns
242 -------
243 ampAssociations : `numpy.ndarray`
244 An N x N matrix (N = number of amplifiers) that illustrates the
245 connections between amplifiers within the detector layout. Each row
246 and column index corresponds to the ampIds of a specific pair of
247 amplifiers, and the matrix elements indicate their associations as
248 follows:
249 0: No association
250 -1: Association exists (direction specified in the ampSides matrix)
251 n >= 1: Diagonal elements indicate the number of neighboring
252 amplifiers for the corresponding ampId==row==column number.
254 ampSides : `numpy.ndarray`
255 An N x N matrix (N = the number of amplifiers) representing the amp
256 side information corresponding to the `ampAssociations`
257 matrix. The elements are integers defined as below:
258 -1: No side due to no association or the same amp (diagonals)
259 0: Side on the bottom
260 1: Side on the right
261 2: Side on the top
262 3: Side on the left
263 """
264 xCenters = [amp.getBBox().getCenterX() for amp in amps]
265 yCenters = [amp.getBBox().getCenterY() for amp in amps]
266 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1
267 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1
269 nAmps = len(amps)
270 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int)
272 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices):
273 ampIds[yIndex, xIndex] = ampId
275 ampAssociations = np.zeros((nAmps, nAmps), dtype=int)
276 ampSides = np.full_like(ampAssociations, -1)
278 for ampId in ampIds.ravel():
279 neighbors, sides = self.getNeighbors(ampIds, ampId)
280 ampAssociations[ampId, neighbors] = -1
281 ampSides[ampId, neighbors] = sides
282 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum()
284 if ampAssociations.sum() != 0:
285 raise RuntimeError("The `ampAssociations` array does not sum to zero.")
287 if not np.all(ampAssociations == ampAssociations.T):
288 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.")
290 self.log.debug("amp associations:\n%s", ampAssociations)
291 self.log.debug("amp sides:\n%s", ampSides)
293 return ampAssociations, ampSides
295 def getNeighbors(self, ampIds, ampId):
296 """Get the neighbor amplifiers and their sides for a given
297 amplifier.
299 Parameters
300 ----------
301 ampIds : `numpy.ndarray`
302 Matrix with amp side association information.
303 ampId : `int`
304 The amplifier ID for which neighbor amplifiers and side IDs
305 are to be found.
307 Returns
308 -------
309 neighbors : `list` [`int`]
310 List of neighbor amplifier IDs.
311 sides : `list` [`int`]
312 List of side IDs, with each ID corresponding to its respective
313 neighbor amplifier.
314 """
315 m, n = ampIds.shape
316 r, c = np.ravel(np.where(ampIds == ampId))
317 neighbors, sides = [], []
318 sideLookup = {
319 0: (r + 1, c),
320 1: (r, c + 1),
321 2: (r - 1, c),
322 3: (r, c - 1),
323 }
324 for side, (row, column) in sideLookup.items():
325 if 0 <= row < m and 0 <= column < n:
326 neighbors.append(ampIds[row][column])
327 sides.append(side)
328 return neighbors, sides
330 def getAmpOffsets(self, im, amps, associations, sides):
331 """Calculate the amp offsets for all amplifiers.
333 Parameters
334 ----------
335 im : `lsst.afw.image._image.ImageF`
336 Amplifier image to extract data from.
337 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
338 List of amplifier objects.
339 associations : numpy.ndarray
340 An N x N matrix containing amp association information, where N is
341 the number of amplifiers.
342 sides : numpy.ndarray
343 An N x N matrix containing amp side information, where N is the
344 number of amplifiers.
346 Returns
347 -------
348 ampsOffsets : `numpy.ndarray`
349 1D float array containing the calculated amp offsets for all
350 amplifiers.
351 """
352 ampsOffsets = np.zeros(len(amps))
353 ampsEdges = self.getAmpEdges(im, amps, sides)
354 interfaceOffsetLookup = {}
355 for ampId, ampAssociations in enumerate(associations):
356 ampNeighbors = np.ravel(np.where(ampAssociations < 0))
357 for ampNeighbor in ampNeighbors:
358 ampSide = sides[ampId][ampNeighbor]
359 edgeA = ampsEdges[ampId][ampSide]
360 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4]
361 if ampId < ampNeighbor:
362 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB)
363 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset
364 else:
365 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"]
366 ampsOffsets[ampId] += interfaceOffset
367 return ampsOffsets
369 def getAmpEdges(self, im, amps, ampSides):
370 """Calculate the amp edges for all amplifiers.
372 Parameters
373 ----------
374 im : `lsst.afw.image._image.ImageF`
375 Amplifier image to extract data from.
376 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
377 List of amplifier objects.
378 ampSides : `numpy.ndarray`
379 An N x N matrix containing amp side information, where N is the
380 number of amplifiers.
382 Returns
383 -------
384 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]]
385 A dictionary containing amp edge(s) for each amplifier,
386 corresponding to one or more potential sides, where each edge is
387 associated with a side. The outer dictionary has integer keys
388 representing amplifier IDs, and the inner dictionary has integer
389 keys representing side IDs for each amplifier and values that are
390 1D arrays of floats representing the 1D medianified strips from the
391 amp image, referred to as "amp edge":
392 {ampID: {sideID: numpy.ndarray}, ...}
393 """
394 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth
395 ampEdges = {}
396 slice_map = {
397 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)),
398 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)),
399 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)),
400 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)),
401 }
402 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)):
403 ampEdges[ampId] = {}
404 ampIm = im[amp.getBBox()].array
405 # Loop over identified sides.
406 for ampSide in ampSides:
407 if ampSide < 0:
408 continue
409 strip = ampIm[slice_map[ampSide]]
410 # Catch warnings to prevent all-NaN slice RuntimeWarning.
411 with warnings.catch_warnings():
412 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered")
413 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip
414 return ampEdges
416 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB):
417 """Calculate the amp offset for a given interface between two
418 amplifiers.
420 Parameters
421 ----------
422 ampIdA : int
423 ID of the first amplifier.
424 ampIdB : int
425 ID of the second amplifier.
426 edgeA : numpy.ndarray
427 Amp edge for the first amplifier.
428 edgeB : numpy.ndarray
429 Amp edge for the second amplifier.
431 Returns
432 -------
433 interfaceOffset : float
434 The calculated amp offset value for the given interface between
435 amps A and B.
436 """
437 interfaceId = f"{ampIdA}{ampIdB}"
438 sctrl = StatisticsControl()
439 # NOTE: Taking the difference with the order below fixes the sign flip
440 # in the B matrix.
441 edgeDiff = edgeA - edgeB
442 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff))
443 # Compute rolling averages.
444 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same")
445 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same")
446 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None)
447 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan
448 # Take clipped mean of rolling average data as amp offset value.
449 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue()
450 # Perform a couple of do-no-harm safety checks:
451 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac,
452 # b) The absolute offset ADU value is < ampEdgeMaxOffset.
453 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg))
454 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac
455 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset
456 if minFracFail or maxOffsetFail:
457 interfaceOffset = 0
458 self.log.warning(
459 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold "
460 f"({self.config.ampEdgeMinFrac}) or the absolute offset value exceeds the limit "
461 f"({self.config.ampEdgeMaxOffset} ADU). Setting the interface offset to 0."
462 )
463 self.log.debug(
464 f"amp interface {interfaceId} : "
465 f"viable edge difference frac = {ampEdgeGoodFrac}, "
466 f"interface offset = {interfaceOffset:.3f}"
467 )
468 return interfaceOffset