lsst.ip.isr ged8ae655b3+c95ea48ad7
Loading...
Searching...
No Matches
ampOffset.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"]
23
24import warnings
25
26import numpy as np
27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics
28from lsst.afw.table import SourceTable
29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask
30from lsst.pex.config import Config, ConfigurableField, Field
31from lsst.pipe.base import Task, Struct
32
33
34class AmpOffsetConfig(Config):
35 """Configuration parameters for AmpOffsetTask."""
36
37 def setDefaults(self):
38 self.background.algorithm = "AKIMA_SPLINE"
39 self.background.useApprox = False
40 self.background.ignoredPixelMask = [
41 "BAD",
42 "SAT",
43 "INTRP",
44 "CR",
45 "EDGE",
46 "DETECTED",
47 "DETECTED_NEGATIVE",
48 "SUSPECT",
49 "NO_DATA",
50 ]
51 self.detection.reEstimateBackground = False
52
53 # This maintains existing behavior and test values after DM-39796.
54 self.detection.thresholdType = "stdev"
55
56 ampEdgeInset = Field(
57 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running "
58 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.",
59 dtype=int,
60 default=5,
61 )
62 ampEdgeWidth = Field(
63 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.",
64 dtype=int,
65 default=64,
66 )
67 ampEdgeMinFrac = Field(
68 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be "
69 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.",
70 dtype=float,
71 default=0.5,
72 )
73 ampEdgeMaxOffset = Field(
74 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the "
75 "result will be discarded and therefore not used to determine amp pedestal corrections.",
76 dtype=float,
77 default=5.0,
78 )
79 ampEdgeWindowFrac = Field(
80 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average "
81 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should "
82 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the "
83 "sliding window used in obs_subaru for compatibility with existing HSC data.",
84 dtype=float,
85 default=512 / 4176,
86 )
87 doBackground = Field(
88 doc="Estimate and subtract background prior to amp offset estimation?",
89 dtype=bool,
90 default=True,
91 )
92 background = ConfigurableField(
93 doc="An initial background estimation step run prior to amp offset calculation.",
94 target=SubtractBackgroundTask,
95 )
96 backgroundFractionSample = Field(
97 doc="The fraction of the shorter side of the amplifier used for background binning.",
98 dtype=float,
99 default=1.0,
100 )
101 doDetection = Field(
102 doc="Detect sources and update cloned exposure prior to amp offset estimation?",
103 dtype=bool,
104 default=True,
105 )
106 detection = ConfigurableField(
107 doc="Source detection to add temporary detection footprints prior to amp offset calculation.",
108 target=SourceDetectionTask,
109 )
110
111
112class AmpOffsetTask(Task):
113 """Calculate and apply amp offset corrections to an exposure."""
114
115 ConfigClass = AmpOffsetConfig
116 _DefaultName = "isrAmpOffset"
117
118 def __init__(self, *args, **kwargs):
119 super().__init__(*args, **kwargs)
120 # Always load background subtask, even if doBackground=False;
121 # this allows for default plane bit masks to be defined.
122 self.makeSubtask("background")
123 if self.config.doDetection:
124 self.makeSubtask("detection")
125 # Initialize all of the instance variables here.
127
128 def run(self, exposure):
129 """Calculate amp offset values, determine corrective pedestals for each
130 amp, and update the input exposure in-place.
131
132 Parameters
133 ----------
134 exposure: `lsst.afw.image.Exposure`
135 Exposure to be corrected for amp offsets.
136 """
137
138 # Generate an exposure clone to work on and establish the bit mask.
139 exp = exposure.clone()
140 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask)
141 amps = exp.getDetector().getAmplifiers()
142
143 # Check that all amps have the same gemotry.
144 ampDims = [amp.getBBox().getDimensions() for amp in amps]
145 if not all(dim == ampDims[0] for dim in ampDims):
146 raise RuntimeError("All amps should have the same geometry.")
147
148 # Assuming all the amps have the same geometry.
149 self.shortAmpSide = np.min(ampDims[0])
150
151 # Fit and subtract background.
152 if self.config.doBackground:
153 maskedImage = exp.getMaskedImage()
154 # Assuming all the detectors are the same.
155 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
156 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
157 # This ensures that the `binSize` is as large as possible,
158 # preventing background subtraction from inadvertently removing the
159 # amp offset signature. Here it's set to the shorter dimension of
160 # the amplifier by default (`backgroundFractionSample` = 1), which
161 # seems reasonable.
162 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY))
163 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
164 maskedImage -= bgImage
165
166 # Detect sources and update cloned exposure mask planes in-place.
167 if self.config.doDetection:
168 schema = SourceTable.makeMinimalSchema()
169 table = SourceTable.make(schema)
170 # Detection sigma, used for smoothing and to grow detections, is
171 # normally measured from the PSF of the exposure. As the PSF hasn't
172 # been measured at this stage of processing, sigma is instead
173 # set to an approximate value here (which should be sufficient).
174 _ = self.detection.run(table=table, exposure=exp, sigma=2)
175
176 # Safety check: do any pixels remain for amp offset estimation?
177 if (exp.mask.array & bitMask).all():
178 self.log.warning(
179 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set "
180 "to zero."
181 )
182 pedestals = np.zeros(len(amps))
183 else:
184 # Set up amp offset inputs.
185 im = exp.image
186 im.array[(exp.mask.array & bitMask) > 0] = np.nan
187
188 if self.config.ampEdgeWindowFrac > 1:
189 raise RuntimeError(
190 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the "
191 "edge length exceeds 1. This leads to complications downstream, after convolution in "
192 "the `getSideAmpOffset()` method. Please modify the `ampEdgeWindowFrac` value in the "
193 "config to be 1 or less and rerun."
194 )
195
196 # Determine amplifier geometry.
197 ampAreas = {amp.getBBox().getArea() for amp in amps}
198 if len(ampAreas) > 1:
199 raise NotImplementedError(
200 "Amp offset correction is not yet implemented for detectors with differing amp sizes."
201 )
202
203 # Obtain association and offset matrices.
204 A, sides = self.getAmpAssociations(amps)
205 B = self.getAmpOffsets(im, amps, A, sides)
206
207 # If least-squares minimization fails, convert NaNs to zeroes,
208 # ensuring that no values are erroneously added/subtracted.
209 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0])
210
211 metadata = exposure.getMetadata()
212 for amp, pedestal in zip(amps, pedestals):
213 ampIm = exposure.image[amp.getBBox()].array
214 ampIm -= pedestal
215 ampName = amp.getName()
216 metadata.set(
217 f"LSST ISR AMPOFFSET PEDESTAL {ampName}",
218 float(pedestal),
219 f"Pedestal level subtracted from amp {ampName}",
220 )
221 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}")
222
223 return Struct(pedestals=pedestals)
224
225 def getAmpAssociations(self, amps):
226 """Determine amp geometry and amp associations from a list of
227 amplifiers.
228
229 Parse an input list of amplifiers to determine the layout of amps
230 within a detector, and identify all amp sides (i.e., the
231 horizontal and vertical junctions between amps).
232
233 Returns a matrix with a shape corresponding to the geometry of the amps
234 in the detector.
235
236 Parameters
237 ----------
238 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
239 List of amplifier objects used to deduce associations.
240
241 Returns
242 -------
243 ampAssociations : `numpy.ndarray`
244 An N x N matrix (N = number of amplifiers) that illustrates the
245 connections between amplifiers within the detector layout. Each row
246 and column index corresponds to the ampIds of a specific pair of
247 amplifiers, and the matrix elements indicate their associations as
248 follows:
249 0: No association
250 -1: Association exists (direction specified in the ampSides matrix)
251 n >= 1: Diagonal elements indicate the number of neighboring
252 amplifiers for the corresponding ampId==row==column number.
253
254 ampSides : `numpy.ndarray`
255 An N x N matrix (N = the number of amplifiers) representing the amp
256 side information corresponding to the `ampAssociations`
257 matrix. The elements are integers defined as below:
258 -1: No side due to no association or the same amp (diagonals)
259 0: Side on the bottom
260 1: Side on the right
261 2: Side on the top
262 3: Side on the left
263 """
264 xCenters = [amp.getBBox().getCenterX() for amp in amps]
265 yCenters = [amp.getBBox().getCenterY() for amp in amps]
266 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1
267 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1
268
269 nAmps = len(amps)
270 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int)
271
272 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices):
273 ampIds[yIndex, xIndex] = ampId
274
275 ampAssociations = np.zeros((nAmps, nAmps), dtype=int)
276 ampSides = np.full_like(ampAssociations, -1)
277
278 for ampId in ampIds.ravel():
279 neighbors, sides = self.getNeighbors(ampIds, ampId)
280 ampAssociations[ampId, neighbors] = -1
281 ampSides[ampId, neighbors] = sides
282 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum()
283
284 if ampAssociations.sum() != 0:
285 raise RuntimeError("The `ampAssociations` array does not sum to zero.")
286
287 if not np.all(ampAssociations == ampAssociations.T):
288 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.")
289
290 self.log.debug("amp associations:\n%s", ampAssociations)
291 self.log.debug("amp sides:\n%s", ampSides)
292
293 return ampAssociations, ampSides
294
295 def getNeighbors(self, ampIds, ampId):
296 """Get the neighbor amplifiers and their sides for a given
297 amplifier.
298
299 Parameters
300 ----------
301 ampIds : `numpy.ndarray`
302 Matrix with amp side association information.
303 ampId : `int`
304 The amplifier ID for which neighbor amplifiers and side IDs
305 are to be found.
306
307 Returns
308 -------
309 neighbors : `list` [`int`]
310 List of neighbor amplifier IDs.
311 sides : `list` [`int`]
312 List of side IDs, with each ID corresponding to its respective
313 neighbor amplifier.
314 """
315 m, n = ampIds.shape
316 r, c = np.ravel(np.where(ampIds == ampId))
317 neighbors, sides = [], []
318 sideLookup = {
319 0: (r + 1, c),
320 1: (r, c + 1),
321 2: (r - 1, c),
322 3: (r, c - 1),
323 }
324 for side, (row, column) in sideLookup.items():
325 if 0 <= row < m and 0 <= column < n:
326 neighbors.append(ampIds[row][column])
327 sides.append(side)
328 return neighbors, sides
329
330 def getAmpOffsets(self, im, amps, associations, sides):
331 """Calculate the amp offsets for all amplifiers.
332
333 Parameters
334 ----------
335 im : `lsst.afw.image._image.ImageF`
336 Amplifier image to extract data from.
337 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
338 List of amplifier objects.
339 associations : numpy.ndarray
340 An N x N matrix containing amp association information, where N is
341 the number of amplifiers.
342 sides : numpy.ndarray
343 An N x N matrix containing amp side information, where N is the
344 number of amplifiers.
345
346 Returns
347 -------
348 ampsOffsets : `numpy.ndarray`
349 1D float array containing the calculated amp offsets for all
350 amplifiers.
351 """
352 ampsOffsets = np.zeros(len(amps))
353 ampsEdges = self.getAmpEdges(im, amps, sides)
354 interfaceOffsetLookup = {}
355 for ampId, ampAssociations in enumerate(associations):
356 ampNeighbors = np.ravel(np.where(ampAssociations < 0))
357 for ampNeighbor in ampNeighbors:
358 ampSide = sides[ampId][ampNeighbor]
359 edgeA = ampsEdges[ampId][ampSide]
360 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4]
361 if ampId < ampNeighbor:
362 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB)
363 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset
364 else:
365 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"]
366 ampsOffsets[ampId] += interfaceOffset
367 return ampsOffsets
368
369 def getAmpEdges(self, im, amps, ampSides):
370 """Calculate the amp edges for all amplifiers.
371
372 Parameters
373 ----------
374 im : `lsst.afw.image._image.ImageF`
375 Amplifier image to extract data from.
376 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
377 List of amplifier objects.
378 ampSides : `numpy.ndarray`
379 An N x N matrix containing amp side information, where N is the
380 number of amplifiers.
381
382 Returns
383 -------
384 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]]
385 A dictionary containing amp edge(s) for each amplifier,
386 corresponding to one or more potential sides, where each edge is
387 associated with a side. The outer dictionary has integer keys
388 representing amplifier IDs, and the inner dictionary has integer
389 keys representing side IDs for each amplifier and values that are
390 1D arrays of floats representing the 1D medianified strips from the
391 amp image, referred to as "amp edge":
392 {ampID: {sideID: numpy.ndarray}, ...}
393 """
394 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth
395 ampEdges = {}
396 slice_map = {
397 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)),
398 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)),
399 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)),
400 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)),
401 }
402 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)):
403 ampEdges[ampId] = {}
404 ampIm = im[amp.getBBox()].array
405 # Loop over identified sides.
406 for ampSide in ampSides:
407 if ampSide < 0:
408 continue
409 strip = ampIm[slice_map[ampSide]]
410 # Catch warnings to prevent all-NaN slice RuntimeWarning.
411 with warnings.catch_warnings():
412 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered")
413 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip
414 return ampEdges
415
416 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB):
417 """Calculate the amp offset for a given interface between two
418 amplifiers.
419
420 Parameters
421 ----------
422 ampIdA : int
423 ID of the first amplifier.
424 ampIdB : int
425 ID of the second amplifier.
426 edgeA : numpy.ndarray
427 Amp edge for the first amplifier.
428 edgeB : numpy.ndarray
429 Amp edge for the second amplifier.
430
431 Returns
432 -------
433 interfaceOffset : float
434 The calculated amp offset value for the given interface between
435 amps A and B.
436 """
437 interfaceId = f"{ampIdA}{ampIdB}"
438 sctrl = StatisticsControl()
439 # NOTE: Taking the difference with the order below fixes the sign flip
440 # in the B matrix.
441 edgeDiff = edgeA - edgeB
442 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff))
443 # Compute rolling averages.
444 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same")
445 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same")
446 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None)
447 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan
448 # Take clipped mean of rolling average data as amp offset value.
449 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue()
450 # Perform a couple of do-no-harm safety checks:
451 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac,
452 # b) The absolute offset ADU value is < ampEdgeMaxOffset.
453 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg))
454 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac
455 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset
456 if minFracFail or maxOffsetFail:
457 interfaceOffset = 0
458 self.log.warning(
459 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold "
460 f"({self.config.ampEdgeMinFrac}) or the absolute offset value exceeds the limit "
461 f"({self.config.ampEdgeMaxOffset} ADU). Setting the interface offset to 0."
462 )
463 self.log.debug(
464 f"amp interface {interfaceId} : "
465 f"viable edge difference frac = {ampEdgeGoodFrac}, "
466 f"interface offset = {interfaceOffset:.3f}"
467 )
468 return interfaceOffset
getAmpOffsets(self, im, amps, associations, sides)
Definition ampOffset.py:330
__init__(self, *args, **kwargs)
Definition ampOffset.py:118
getAmpEdges(self, im, amps, ampSides)
Definition ampOffset.py:369
getNeighbors(self, ampIds, ampId)
Definition ampOffset.py:295
getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB)
Definition ampOffset.py:416