lsst.ip.isr g44050f54f7+b04a903948
Loading...
Searching...
No Matches
ampOffset.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["AmpOffsetConfig", "AmpOffsetTask"]
23
24import warnings
25
26import numpy as np
27from lsst.afw.math import MEANCLIP, StatisticsControl, makeStatistics
28from lsst.afw.table import SourceTable
29from lsst.meas.algorithms import SourceDetectionTask, SubtractBackgroundTask
30from lsst.pex.config import Config, ConfigurableField, Field
31from lsst.pipe.base import Struct, Task
32
33
34class AmpOffsetConfig(Config):
35 """Configuration parameters for AmpOffsetTask."""
36
37 def setDefaults(self):
38 self.background.algorithm = "AKIMA_SPLINE"
39 self.background.useApprox = False
40 self.background.ignoredPixelMask = [
41 "BAD",
42 "SAT",
43 "INTRP",
44 "CR",
45 "EDGE",
46 "DETECTED",
47 "DETECTED_NEGATIVE",
48 "SUSPECT",
49 "NO_DATA",
50 ]
51 self.detection.reEstimateBackground = False
52
53 # This maintains existing behavior and test values after DM-39796.
54 self.detection.thresholdType = "stdev"
55
56 ampEdgeInset = Field(
57 doc="Number of pixels the amp edge strip is inset from the amp edge. A thin strip of pixels running "
58 "parallel to the edge of the amp is used to characterize the average flux level at the amp edge.",
59 dtype=int,
60 default=5,
61 )
62 ampEdgeWidth = Field(
63 doc="Pixel width of the amp edge strip, starting at ampEdgeInset and extending inwards.",
64 dtype=int,
65 default=64,
66 )
67 ampEdgeMinFrac = Field(
68 doc="Minimum allowed fraction of viable pixel rows along an amp edge. No amp offset estimate will be "
69 "generated for amp edges that do not have at least this fraction of unmasked pixel rows.",
70 dtype=float,
71 default=0.5,
72 )
73 ampEdgeMaxOffset = Field(
74 doc="Maximum allowed amp offset ADU value. If a measured amp offset value is larger than this, the "
75 "result will be discarded and therefore not used to determine amp pedestal corrections.",
76 dtype=float,
77 default=5.0,
78 )
79 ampEdgeWindowFrac = Field(
80 doc="Fraction of the amp edge lengths utilized as the sliding window for generating rolling average "
81 "amp offset values. It should be reconfigured for every instrument (HSC, LSSTCam, etc.) and should "
82 "not exceed 1. If not provided, it defaults to the fraction that recovers the pixel size of the "
83 "sliding window used in obs_subaru for compatibility with existing HSC data.",
84 dtype=float,
85 default=512 / 4176,
86 )
87 doBackground = Field(
88 doc="Estimate and subtract background prior to amp offset estimation?",
89 dtype=bool,
90 default=True,
91 )
92 background = ConfigurableField(
93 doc="An initial background estimation step run prior to amp offset calculation.",
94 target=SubtractBackgroundTask,
95 )
96 backgroundFractionSample = Field(
97 doc="The fraction of the shorter side of the amplifier used for background binning.",
98 dtype=float,
99 default=1.0,
100 )
101 doDetection = Field(
102 doc="Detect sources and update cloned exposure prior to amp offset estimation?",
103 dtype=bool,
104 default=True,
105 )
106 detection = ConfigurableField(
107 doc="Source detection to add temporary detection footprints prior to amp offset calculation.",
108 target=SourceDetectionTask,
109 )
110 applyWeights = Field(
111 doc="Weights the amp offset calculation by the length of the interface between amplifiers. Applying "
112 "weights does not affect outcomes for amplifiers in a 2D grid with square-shaped amplifiers or in "
113 "any 1D layout on a detector, regardless of whether the amplifiers are square.",
114 dtype=bool,
115 default=True,
116 )
117
118
119class AmpOffsetTask(Task):
120 """Calculate and apply amp offset corrections to an exposure."""
121
122 ConfigClass = AmpOffsetConfig
123 _DefaultName = "isrAmpOffset"
124
125 def __init__(self, *args, **kwargs):
126 super().__init__(*args, **kwargs)
127 # Always load background subtask, even if doBackground=False;
128 # this allows for default plane bit masks to be defined.
129 self.makeSubtask("background")
130 if self.config.doDetection:
131 self.makeSubtask("detection")
132 # Initialize all of the instance variables here.
134
135 def run(self, exposure):
136 """Calculate amp offset values, determine corrective pedestals for each
137 amp, and update the input exposure in-place.
138
139 Parameters
140 ----------
141 exposure: `lsst.afw.image.Exposure`
142 Exposure to be corrected for amp offsets.
143 """
144
145 # Generate an exposure clone to work on and establish the bit mask.
146 exp = exposure.clone()
147 bitMask = exp.mask.getPlaneBitMask(self.background.config.ignoredPixelMask)
148 amps = exp.getDetector().getAmplifiers()
149
150 # Check that all amps have the same gemotry.
151 ampDims = [amp.getBBox().getDimensions() for amp in amps]
152 if not all(dim == ampDims[0] for dim in ampDims):
153 raise RuntimeError("All amps should have the same geometry.")
154 else:
155 # The zeroth amp is representative of all amps in the detector.
156 self.ampDims = ampDims[0]
157 # Dictionary mapping side numbers to interface lengths.
158 # See `getAmpAssociations()` for details about sides.
159 self.interfaceLengthLookupBySide = {i: self.ampDims[i % 2] for i in range(4)}
160
161 # Determine amplifier geometry.
162 ampWidths = {amp.getBBox().getWidth() for amp in amps}
163 ampHeights = {amp.getBBox().getHeight() for amp in amps}
164 if len(ampWidths) > 1 or len(ampHeights) > 1:
165 raise NotImplementedError(
166 "Amp offset correction is not yet implemented for detectors with differing amp sizes."
167 )
168
169 # Assuming all the amps have the same geometry.
170 self.shortAmpSide = np.min(ampDims[0])
171
172 # Check that the edge width and inset are not too large.
173 if self.config.ampEdgeWidth >= self.shortAmpSide - 2 * self.config.ampEdgeInset:
174 raise RuntimeError(
175 f"The edge width ({self.config.ampEdgeWidth}) plus insets ({self.config.ampEdgeInset}) "
176 f"exceed the amp's short side ({self.shortAmpSide}). This setup leads to incorrect results."
177 )
178
179 # Fit and subtract background.
180 if self.config.doBackground:
181 maskedImage = exp.getMaskedImage()
182 # Assuming all the detectors are the same.
183 nX = exp.getWidth() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
184 nY = exp.getHeight() // (self.shortAmpSide * self.config.backgroundFractionSample) + 1
185 # This ensures that the `binSize` is as large as possible,
186 # preventing background subtraction from inadvertently removing the
187 # amp offset signature. Here it's set to the shorter dimension of
188 # the amplifier by default (`backgroundFractionSample` = 1), which
189 # seems reasonable.
190 bg = self.background.fitBackground(maskedImage, nx=int(nX), ny=int(nY))
191 bgImage = bg.getImageF(self.background.config.algorithm, self.background.config.undersampleStyle)
192 maskedImage -= bgImage
193
194 # Detect sources and update cloned exposure mask planes in-place.
195 if self.config.doDetection:
196 schema = SourceTable.makeMinimalSchema()
197 table = SourceTable.make(schema)
198 # Detection sigma, used for smoothing and to grow detections, is
199 # normally measured from the PSF of the exposure. As the PSF hasn't
200 # been measured at this stage of processing, sigma is instead
201 # set to an approximate value here (which should be sufficient).
202 _ = self.detection.run(table=table, exposure=exp, sigma=2)
203
204 # Safety check: do any pixels remain for amp offset estimation?
205 if (exp.mask.array & bitMask).all():
206 self.log.warning(
207 "All pixels masked: cannot calculate any amp offset corrections. All pedestals are being set "
208 "to zero."
209 )
210 pedestals = np.zeros(len(amps))
211 else:
212 # Set up amp offset inputs.
213 im = exp.image
214 im.array[(exp.mask.array & bitMask) > 0] = np.nan
215
216 if self.config.ampEdgeWindowFrac > 1:
217 raise RuntimeError(
218 f"The specified fraction (`ampEdgeWindowFrac`={self.config.ampEdgeWindowFrac}) of the "
219 "edge length exceeds 1. This leads to complications downstream, after convolution in "
220 "the `getInterfaceOffset()` method. Please modify the `ampEdgeWindowFrac` value in the "
221 "config to be 1 or less and rerun."
222 )
223
224 # Obtain association and offset matrices.
225 A, sides = self.getAmpAssociations(amps)
226 B = self.getAmpOffsets(im, amps, A, sides)
227
228 # If least-squares minimization fails, convert NaNs to zeroes,
229 # ensuring that no values are erroneously added/subtracted.
230 pedestals = np.nan_to_num(np.linalg.lstsq(A, B, rcond=None)[0])
231
232 metadata = exposure.getMetadata()
233 for amp, pedestal in zip(amps, pedestals):
234 ampIm = exposure.image[amp.getBBox()].array
235 ampIm -= pedestal
236 ampName = amp.getName()
237 metadata.set(
238 f"LSST ISR AMPOFFSET PEDESTAL {ampName}",
239 float(pedestal),
240 f"Pedestal level subtracted from amp {ampName}",
241 )
242 self.log.info(f"amp pedestal values: {', '.join([f'{x:.4f}' for x in pedestals])}")
243
244 return Struct(pedestals=pedestals)
245
246 def getAmpAssociations(self, amps):
247 """Determine amp geometry and amp associations from a list of
248 amplifiers.
249
250 Parse an input list of amplifiers to determine the layout of amps
251 within a detector, and identify all amp sides (i.e., the
252 horizontal and vertical junctions between amps).
253
254 Returns a matrix with a shape corresponding to the geometry of the amps
255 in the detector.
256
257 Parameters
258 ----------
259 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
260 List of amplifier objects used to deduce associations.
261
262 Returns
263 -------
264 ampAssociations : `numpy.ndarray`
265 An N x N matrix (N = number of amplifiers) that illustrates the
266 connections between amplifiers within the detector layout. Each row
267 and column index corresponds to the ampIds of a specific pair of
268 amplifiers, and the matrix elements indicate their associations as
269 follows:
270 0: No association
271 -1: Association exists (direction specified in the ampSides matrix)
272 n >= 1: Diagonal elements indicate the number of neighboring
273 amplifiers for the corresponding ampId==row==column number.
274
275 ampSides : `numpy.ndarray`
276 An N x N matrix (N = the number of amplifiers) representing the amp
277 side information corresponding to the `ampAssociations`
278 matrix. The elements are integers defined as below:
279 -1: No side due to no association or the same amp (diagonals)
280 0: Side on the bottom
281 1: Side on the right
282 2: Side on the top
283 3: Side on the left
284 """
285 xCenters = [amp.getBBox().getCenterX() for amp in amps]
286 yCenters = [amp.getBBox().getCenterY() for amp in amps]
287 xIndices = np.ceil(xCenters / np.min(xCenters) / 2).astype(int) - 1
288 yIndices = np.ceil(yCenters / np.min(yCenters) / 2).astype(int) - 1
289
290 nAmps = len(amps)
291 ampIds = np.zeros((len(set(yIndices)), len(set(xIndices))), dtype=int)
292
293 for ampId, xIndex, yIndex in zip(np.arange(nAmps), xIndices, yIndices):
294 ampIds[yIndex, xIndex] = ampId
295
296 ampAssociations = np.zeros((nAmps, nAmps), dtype=int)
297 ampSides = np.full_like(ampAssociations, -1)
298
299 for ampId in ampIds.ravel():
300 neighbors, sides = self.getNeighbors(ampIds, ampId)
301 interfaceWeights = (
302 1
303 if not self.config.applyWeights
304 else np.array([self.interfaceLengthLookupBySide[side] for side in sides])
305 )
306 ampAssociations[ampId, neighbors] = -1 * interfaceWeights
307 ampSides[ampId, neighbors] = sides
308 ampAssociations[ampId, ampId] = -ampAssociations[ampId].sum()
309
310 if ampAssociations.sum() != 0:
311 raise RuntimeError("The `ampAssociations` array does not sum to zero.")
312
313 if not np.all(ampAssociations == ampAssociations.T):
314 raise RuntimeError("The `ampAssociations` is not symmetric about the diagonal.")
315
316 self.log.debug("amp associations:\n%s", ampAssociations)
317 self.log.debug("amp sides:\n%s", ampSides)
318
319 return ampAssociations, ampSides
320
321 def getNeighbors(self, ampIds, ampId):
322 """Get the neighbor amplifiers and their sides for a given
323 amplifier.
324
325 Parameters
326 ----------
327 ampIds : `numpy.ndarray`
328 Matrix with amp side association information.
329 ampId : `int`
330 The amplifier ID for which neighbor amplifiers and side IDs
331 are to be found.
332
333 Returns
334 -------
335 neighbors : `list` [`int`]
336 List of neighbor amplifier IDs.
337 sides : `list` [`int`]
338 List of side IDs, with each ID corresponding to its respective
339 neighbor amplifier.
340 """
341 m, n = ampIds.shape
342 r, c = np.ravel(np.where(ampIds == ampId))
343 neighbors, sides = [], []
344 sideLookup = {
345 0: (r + 1, c),
346 1: (r, c + 1),
347 2: (r - 1, c),
348 3: (r, c - 1),
349 }
350 for side, (row, column) in sideLookup.items():
351 if 0 <= row < m and 0 <= column < n:
352 neighbors.append(ampIds[row][column])
353 sides.append(side)
354 return neighbors, sides
355
356 def getAmpOffsets(self, im, amps, associations, sides):
357 """Calculate the amp offsets for all amplifiers.
358
359 Parameters
360 ----------
361 im : `lsst.afw.image._image.ImageF`
362 Amplifier image to extract data from.
363 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
364 List of amplifier objects.
365 associations : numpy.ndarray
366 An N x N matrix containing amp association information, where N is
367 the number of amplifiers.
368 sides : numpy.ndarray
369 An N x N matrix containing amp side information, where N is the
370 number of amplifiers.
371
372 Returns
373 -------
374 ampsOffsets : `numpy.ndarray`
375 1D float array containing the calculated amp offsets for all
376 amplifiers.
377 """
378 ampsOffsets = np.zeros(len(amps))
379 ampsEdges = self.getAmpEdges(im, amps, sides)
380 interfaceOffsetLookup = {}
381
382 for ampId, ampAssociations in enumerate(associations):
383 ampNeighbors = np.ravel(np.where(ampAssociations < 0))
384 for ampNeighbor in ampNeighbors:
385 ampSide = sides[ampId][ampNeighbor]
386 interfaceWeight = (
387 1 if not self.config.applyWeights else self.interfaceLengthLookupBySide[ampSide]
388 )
389 edgeA = ampsEdges[ampId][ampSide]
390 edgeB = ampsEdges[ampNeighbor][(ampSide + 2) % 4]
391 if ampId < ampNeighbor:
392 interfaceOffset = self.getInterfaceOffset(ampId, ampNeighbor, edgeA, edgeB)
393 interfaceOffsetLookup[f"{ampId}{ampNeighbor}"] = interfaceOffset
394 else:
395 interfaceOffset = -interfaceOffsetLookup[f"{ampNeighbor}{ampId}"]
396 ampsOffsets[ampId] += interfaceWeight * interfaceOffset
397 return ampsOffsets
398
399 def getAmpEdges(self, im, amps, ampSides):
400 """Calculate the amp edges for all amplifiers.
401
402 Parameters
403 ----------
404 im : `lsst.afw.image._image.ImageF`
405 Amplifier image to extract data from.
406 amps : `list` [`lsst.afw.cameraGeom.Amplifier`]
407 List of amplifier objects.
408 ampSides : `numpy.ndarray`
409 An N x N matrix containing amp side information, where N is the
410 number of amplifiers.
411
412 Returns
413 -------
414 ampEdges : `dict` [`int`, `dict` [`int`, `numpy.ndarray`]]
415 A dictionary containing amp edge(s) for each amplifier,
416 corresponding to one or more potential sides, where each edge is
417 associated with a side. The outer dictionary has integer keys
418 representing amplifier IDs, and the inner dictionary has integer
419 keys representing side IDs for each amplifier and values that are
420 1D arrays of floats representing the 1D medianified strips from the
421 amp image, referred to as "amp edge":
422 {ampID: {sideID: numpy.ndarray}, ...}
423 """
424 ampEdgeOuter = self.config.ampEdgeInset + self.config.ampEdgeWidth
425 ampEdges = {}
426 slice_map = {
427 0: (slice(-ampEdgeOuter, -self.config.ampEdgeInset), slice(None)),
428 1: (slice(None), slice(-ampEdgeOuter, -self.config.ampEdgeInset)),
429 2: (slice(self.config.ampEdgeInset, ampEdgeOuter), slice(None)),
430 3: (slice(None), slice(self.config.ampEdgeInset, ampEdgeOuter)),
431 }
432 for ampId, (amp, ampSides) in enumerate(zip(amps, ampSides)):
433 ampEdges[ampId] = {}
434 ampIm = im[amp.getBBox()].array
435 # Loop over identified sides.
436 for ampSide in ampSides:
437 if ampSide < 0:
438 continue
439 strip = ampIm[slice_map[ampSide]]
440 # Catch warnings to prevent all-NaN slice RuntimeWarning.
441 with warnings.catch_warnings():
442 warnings.filterwarnings("ignore", r"All-NaN (slice|axis) encountered")
443 ampEdges[ampId][ampSide] = np.nanmedian(strip, axis=ampSide % 2) # 1D medianified strip
444 return ampEdges
445
446 def getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB):
447 """Calculate the amp offset for a given interface between two
448 amplifiers.
449
450 Parameters
451 ----------
452 ampIdA : int
453 ID of the first amplifier.
454 ampIdB : int
455 ID of the second amplifier.
456 edgeA : numpy.ndarray
457 Amp edge for the first amplifier.
458 edgeB : numpy.ndarray
459 Amp edge for the second amplifier.
460
461 Returns
462 -------
463 interfaceOffset : float
464 The calculated amp offset value for the given interface between
465 amps A and B.
466 """
467 interfaceId = f"{ampIdA}{ampIdB}"
468 sctrl = StatisticsControl()
469 # NOTE: Taking the difference with the order below fixes the sign flip
470 # in the B matrix.
471 edgeDiff = edgeA - edgeB
472 window = int(self.config.ampEdgeWindowFrac * len(edgeDiff))
473 # Compute rolling averages.
474 edgeDiffSum = np.convolve(np.nan_to_num(edgeDiff), np.ones(window), "same")
475 edgeDiffNum = np.convolve(~np.isnan(edgeDiff), np.ones(window), "same")
476 edgeDiffAvg = edgeDiffSum / np.clip(edgeDiffNum, 1, None)
477 edgeDiffAvg[np.isnan(edgeDiff)] = np.nan
478 # Take clipped mean of rolling average data as amp offset value.
479 interfaceOffset = makeStatistics(edgeDiffAvg, MEANCLIP, sctrl).getValue()
480 # Perform a couple of do-no-harm safety checks:
481 # a) The fraction of unmasked pixel rows is > ampEdgeMinFrac,
482 # b) The absolute offset ADU value is < ampEdgeMaxOffset.
483 ampEdgeGoodFrac = 1 - (np.sum(np.isnan(edgeDiffAvg)) / len(edgeDiffAvg))
484 minFracFail = ampEdgeGoodFrac < self.config.ampEdgeMinFrac
485 maxOffsetFail = np.abs(interfaceOffset) > self.config.ampEdgeMaxOffset
486 if minFracFail or maxOffsetFail:
487 interfaceOffset = 0
488 if minFracFail:
489 self.log.warning(
490 f"The fraction of unmasked pixels for amp interface {interfaceId} is below the threshold "
491 f"({ampEdgeGoodFrac:.2f} < {self.config.ampEdgeMinFrac}). Setting the interface offset "
492 f"to {interfaceOffset}."
493 )
494 if maxOffsetFail:
495 self.log.warning(
496 "The absolute offset value exceeds the limit "
497 f"({np.abs(interfaceOffset):.2f} > {self.config.ampEdgeMaxOffset} ADU). Setting the "
498 f"interface offset to {interfaceOffset}."
499 )
500 self.log.debug(
501 f"amp interface {interfaceId} : "
502 f"viable edge difference frac = {ampEdgeGoodFrac}, "
503 f"interface offset = {interfaceOffset:.3f}"
504 )
505 return interfaceOffset
getAmpOffsets(self, im, amps, associations, sides)
Definition ampOffset.py:356
__init__(self, *args, **kwargs)
Definition ampOffset.py:125
getAmpEdges(self, im, amps, ampSides)
Definition ampOffset.py:399
getNeighbors(self, ampIds, ampId)
Definition ampOffset.py:321
getInterfaceOffset(self, ampIdA, ampIdB, edgeA, edgeB)
Definition ampOffset.py:446