Coverage for python/lsst/cp/pipe/defects.py: 18%
432 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-15 02:24 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-15 02:24 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask',
24 'MergeDefectsTaskConfig', 'MergeDefectsTask',
25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask',
26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ]
28import numpy as np
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as cT
33from lsstDebug import getDebugFrame
34import lsst.pex.config as pexConfig
36import lsst.afw.image as afwImage
37import lsst.afw.math as afwMath
38import lsst.afw.detection as afwDetection
39import lsst.afw.display as afwDisplay
40from lsst.afw import cameraGeom
41from lsst.geom import Box2I, Point2I, Extent2I
42from lsst.meas.algorithms import SourceDetectionTask
43from lsst.ip.isr import Defects, countMaskedPixels
44from lsst.pex.exceptions import InvalidParameterError
47class MeasureDefectsConnections(pipeBase.PipelineTaskConnections,
48 dimensions=("instrument", "exposure", "detector")):
49 inputExp = cT.Input(
50 name="defectExps",
51 doc="Input ISR-processed exposures to measure.",
52 storageClass="Exposure",
53 dimensions=("instrument", "detector", "exposure"),
54 multiple=False
55 )
56 camera = cT.PrerequisiteInput(
57 name='camera',
58 doc="Camera associated with this exposure.",
59 storageClass="Camera",
60 dimensions=("instrument", ),
61 isCalibration=True,
62 )
64 outputDefects = cT.Output(
65 name="singleExpDefects",
66 doc="Output measured defects.",
67 storageClass="Defects",
68 dimensions=("instrument", "detector", "exposure"),
69 )
72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig,
73 pipelineConnections=MeasureDefectsConnections):
74 """Configuration for measuring defects from a list of exposures
75 """
77 thresholdType = pexConfig.ChoiceField(
78 dtype=str,
79 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found "
80 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found "
81 "in flats, and hot pixels in darks."),
82 default='STDEV',
83 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.",
84 'VALUE': "Use pixel value to determine detection threshold."},
85 )
86 darkCurrentThreshold = pexConfig.Field(
87 dtype=float,
88 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define "
89 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."),
90 default=5,
91 )
92 biasThreshold = pexConfig.Field(
93 dtype=float,
94 doc=("If thresholdType==``VALUE``, bias threshold (in ADU) to define "
95 "hot/bright pixels in bias frame. Unused if thresholdType==``STDEV``."),
96 default=1000.0,
97 )
98 fracThresholdFlat = pexConfig.Field(
99 dtype=float,
100 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark "
101 "pixels in flat images (fraction of the mean value per amplifier)."
102 "Unused if thresholdType==``STDEV``."),
103 default=0.8,
104 )
105 nSigmaBright = pexConfig.Field(
106 dtype=float,
107 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot "
108 "pixel detection. The default value was found to be "
109 "appropriate for some LSST sensors in DM-17490. "
110 "Unused if thresholdType==``VALUE``"),
111 default=4.8,
112 )
113 nSigmaDark = pexConfig.Field(
114 dtype=float,
115 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel "
116 "detection. The default value was found to be "
117 "appropriate for some LSST sensors in DM-17490. "
118 "Unused if thresholdType==``VALUE``"),
119 default=-5.0,
120 )
121 nPixBorderUpDown = pexConfig.Field(
122 dtype=int,
123 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
124 default=0,
125 )
126 nPixBorderLeftRight = pexConfig.Field(
127 dtype=int,
128 doc="Number of pixels to exclude from left & right of image when looking for defects.",
129 default=0,
130 )
131 badOnAndOffPixelColumnThreshold = pexConfig.Field(
132 dtype=int,
133 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) "
134 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the "
135 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long "
136 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all "
137 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive "
138 "non-bad pixels in BPC, an exception to the above is made and those consecutive "
139 "'goodPixelColumnGapThreshold' are not marked as bad."),
140 default=50,
141 )
142 goodPixelColumnGapThreshold = pexConfig.Field(
143 dtype=int,
144 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see "
145 "'badOnAndOffPixelColumnThreshold')."),
146 default=30,
147 )
148 badPixelsToFillColumnThreshold = pexConfig.Field(
149 dtype=float,
150 doc=("If the number of bad pixels in an amplifier column is above this threshold "
151 "then the full amplifier column will be marked bad. This operation is performed after "
152 "any merging of blinking columns performed with badOnAndOffPixelColumnThreshold. If this"
153 "value is less than 0 then no bad column filling will be performed."),
154 default=-1,
155 )
156 saturatedColumnMask = pexConfig.Field(
157 dtype=str,
158 default="SAT",
159 doc="Saturated mask plane for dilation.",
160 )
161 saturatedColumnDilationRadius = pexConfig.Field(
162 dtype=int,
163 doc=("Dilation radius (along rows) to use to expand saturated columns "
164 "to mitigate glow."),
165 default=0,
166 )
167 saturatedPixelsToFillColumnThreshold = pexConfig.Field(
168 dtype=int,
169 doc=("If the number of saturated pixels in an amplifier column is above this threshold "
170 "then the full amplifier column will be marked bad. If this value is less than 0"
171 "then no saturated column filling will be performed."),
172 default=-1,
173 )
175 def validate(self):
176 super().validate()
177 if self.nSigmaBright < 0.0:
178 raise ValueError("nSigmaBright must be above 0.0.")
179 if self.nSigmaDark > 0.0:
180 raise ValueError("nSigmaDark must be below 0.0.")
183class MeasureDefectsTask(pipeBase.PipelineTask):
184 """Measure the defects from one exposure.
185 """
187 ConfigClass = MeasureDefectsTaskConfig
188 _DefaultName = 'cpDefectMeasure'
190 def run(self, inputExp, camera):
191 """Measure one exposure for defects.
193 Parameters
194 ----------
195 inputExp : `lsst.afw.image.Exposure`
196 Exposure to examine.
197 camera : `lsst.afw.cameraGeom.Camera`
198 Camera to use for metadata.
200 Returns
201 -------
202 results : `lsst.pipe.base.Struct`
203 Results struct containing:
205 ``outputDefects``
206 The defects measured from this exposure
207 (`lsst.ip.isr.Defects`).
208 """
209 detector = inputExp.getDetector()
210 try:
211 filterName = inputExp.getFilter().physicalLabel
212 except AttributeError:
213 filterName = None
215 defects = self._findHotAndColdPixels(inputExp)
217 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN')
218 msg = "Found %s defects containing %s pixels in %s"
219 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
221 defects.updateMetadataFromExposures([inputExp])
222 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName,
223 setCalibId=True, setDate=True,
224 cpDefectGenImageType=datasetType)
226 return pipeBase.Struct(
227 outputDefects=defects,
228 )
230 @staticmethod
231 def _nPixFromDefects(defects):
232 """Count pixels in a defect.
234 Parameters
235 ----------
236 defects : `lsst.ip.isr.Defects`
237 Defects to measure.
239 Returns
240 -------
241 nPix : `int`
242 Number of defect pixels.
243 """
244 nPix = 0
245 for defect in defects:
246 nPix += defect.getBBox().getArea()
247 return nPix
249 def _findHotAndColdPixels(self, exp):
250 """Find hot and cold pixels in an image.
252 Using config-defined thresholds on a per-amp basis, mask
253 pixels that are nSigma above threshold in dark frames (hot
254 pixels), or nSigma away from the clipped mean in flats (hot &
255 cold pixels).
257 Parameters
258 ----------
259 exp : `lsst.afw.image.exposure.Exposure`
260 The exposure in which to find defects.
262 Returns
263 -------
264 defects : `lsst.ip.isr.Defects`
265 The defects found in the image.
266 """
267 self._setEdgeBits(exp)
268 maskedIm = exp.maskedImage
270 # the detection polarity for afwDetection, True for positive,
271 # False for negative, and therefore True for darks as they only have
272 # bright pixels, and both for flats, as they have bright and dark pix
273 footprintList = []
275 hotPixelCount = {}
276 coldPixelCount = {}
278 for amp in exp.getDetector():
279 ampName = amp.getName()
281 hotPixelCount[ampName] = 0
282 coldPixelCount[ampName] = 0
284 ampImg = maskedIm[amp.getBBox()].clone()
286 # crop ampImage depending on where the amp lies in the image
287 if self.config.nPixBorderLeftRight:
288 if ampImg.getX0() == 0:
289 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
290 else:
291 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
292 if self.config.nPixBorderUpDown:
293 if ampImg.getY0() == 0:
294 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
295 else:
296 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
298 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
299 continue
301 # Remove a background estimate
302 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
303 ampImg -= meanClip
305 # Determine thresholds
306 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue()
307 expTime = exp.getInfo().getVisitInfo().getExposureTime()
308 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN')
309 if np.isnan(expTime):
310 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second",
311 expTime, ampName, datasetType)
312 expTime = 1.
313 thresholdType = self.config.thresholdType
314 if thresholdType == 'VALUE':
315 # LCA-128 and eoTest: bright/hot pixels in dark images are
316 # defined as any pixel with more than 5 e-/s of dark current.
317 # We scale by the exposure time.
318 if datasetType.lower() == 'dark':
319 # hot pixel threshold
320 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain()
321 elif datasetType.lower() == 'bias':
322 # hot pixel threshold, no exposure time.
323 valueThreshold = self.config.biasThreshold
324 else:
325 # LCA-128 and eoTest: dark/cold pixels in flat images as
326 # defined as any pixel with photoresponse <80% of
327 # the mean (at 500nm).
329 # We subtracted the mean above, so the threshold will be
330 # negative cold pixel threshold.
331 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip
332 # Find equivalent sigma values.
333 if stDev == 0.0:
334 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.",
335 stDev, ampName, datasetType)
336 nSigmaList = [np.inf]
337 else:
338 nSigmaList = [valueThreshold/stDev]
339 else:
340 hotPixelThreshold = self.config.nSigmaBright
341 coldPixelThreshold = self.config.nSigmaDark
342 if datasetType.lower() == 'dark':
343 nSigmaList = [hotPixelThreshold]
344 valueThreshold = stDev*hotPixelThreshold
345 elif datasetType.lower() == 'bias':
346 self.log.warning(
347 "Bias frame detected, but thresholdType == STDEV; not looking for defects.",
348 )
349 return Defects.fromFootprintList([])
350 else:
351 nSigmaList = [hotPixelThreshold, coldPixelThreshold]
352 valueThreshold = [x*stDev for x in nSigmaList]
354 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel"
355 "Values (hot and cold pixels thresholds): %s, %s",
356 datasetType, ampName, thresholdType, nSigmaList, valueThreshold)
357 mergedSet = None
358 for sigma in nSigmaList:
359 nSig = np.abs(sigma)
360 self.debugHistogram('ampFlux', ampImg, nSig, exp)
361 polarity = {-1: False, 1: True}[np.sign(sigma)]
363 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
365 try:
366 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
367 except InvalidParameterError:
368 # This occurs if the image sigma value is 0.0.
369 # Let's mask the whole area.
370 minValue = np.nanmin(ampImg.image.array) - 1.0
371 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True)
372 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
374 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
376 if mergedSet is None:
377 mergedSet = footprintSet
378 else:
379 mergedSet.merge(footprintSet)
381 if polarity:
382 # hot pixels
383 for fp in footprintSet.getFootprints():
384 hotPixelCount[ampName] += fp.getArea()
385 else:
386 # cold pixels
387 for fp in footprintSet.getFootprints():
388 coldPixelCount[ampName] += fp.getArea()
390 footprintList += mergedSet.getFootprints()
392 self.debugView('defectMap', ampImg,
393 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector())
395 defects = Defects.fromFootprintList(footprintList)
396 defects = self.dilateSaturatedColumns(exp, defects)
397 defects, _ = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
398 defects, count = self.maskBadColumns(exp, defects)
399 # We want this to reflect the number of completely bad columns.
400 defects.updateCounters(columns=count, hot=hotPixelCount, cold=coldPixelCount)
402 return defects
404 @staticmethod
405 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
406 """Return the number of non-bad pixels in the image."""
407 nPixels = maskedIm.mask.array.size
408 nBad = countMaskedPixels(maskedIm, badMaskString)
409 return nPixels - nBad
411 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
412 """Set edge bits on an exposure or maskedImage.
414 Raises
415 ------
416 TypeError
417 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
418 """
419 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
420 mi = exposureOrMaskedImage.maskedImage
421 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
422 mi = exposureOrMaskedImage
423 else:
424 t = type(exposureOrMaskedImage)
425 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
427 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
428 if self.config.nPixBorderLeftRight:
429 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
430 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
431 if self.config.nPixBorderUpDown:
432 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
433 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
435 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
436 """Mask blocks in a column if there are on-and-off bad pixels
438 If there's a column with on and off bad pixels, mask all the
439 pixels in between, except if there is a large enough gap of
440 consecutive good pixels between two bad pixels in the column.
442 Parameters
443 ----------
444 defects : `lsst.ip.isr.Defects`
445 The defects found in the image so far
447 Returns
448 -------
449 defects : `lsst.ip.isr.Defects`
450 If the number of bad pixels in a column is not larger or
451 equal than self.config.badPixelColumnThreshold, the input
452 list is returned. Otherwise, the defects list returned
453 will include boxes that mask blocks of on-and-of pixels.
454 badColumnCount : `int`
455 Number of bad columns partially masked.
456 """
457 badColumnCount = 0
458 # Get the (x, y) values of each bad pixel in amp.
459 coordinates = []
460 for defect in defects:
461 bbox = defect.getBBox()
462 x0, y0 = bbox.getMinX(), bbox.getMinY()
463 deltaX0, deltaY0 = bbox.getDimensions()
464 for j in np.arange(y0, y0+deltaY0):
465 for i in np.arange(x0, x0 + deltaX0):
466 coordinates.append((i, j))
468 x, y = [], []
469 for coordinatePair in coordinates:
470 x.append(coordinatePair[0])
471 y.append(coordinatePair[1])
473 x = np.array(x)
474 y = np.array(y)
475 # Find the defects with same "x" (vertical) coordinate (column).
476 unique, counts = np.unique(x, return_counts=True)
477 multipleX = []
478 for (a, b) in zip(unique, counts):
479 if b >= self.config.badOnAndOffPixelColumnThreshold:
480 multipleX.append(a)
481 if len(multipleX) != 0:
482 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
483 badColumnCount += 1
485 return defects, badColumnCount
487 def dilateSaturatedColumns(self, exp, defects):
488 """Dilate saturated columns by a configurable amount.
490 Parameters
491 ----------
492 exp : `lsst.afw.image.exposure.Exposure`
493 The exposure in which to find defects.
494 defects : `lsst.ip.isr.Defects`
495 The defects found in the image so far
497 Returns
498 -------
499 defects : `lsst.ip.isr.Defects`
500 The expanded defects.
501 """
502 if self.config.saturatedColumnDilationRadius <= 0:
503 # This is a no-op.
504 return defects
506 mask = afwImage.Mask.getPlaneBitMask(self.config.saturatedColumnMask)
508 satY, satX = np.where((exp.mask.array & mask) > 0)
510 if len(satX) == 0:
511 # No saturated pixels, nothing to do.
512 return defects
514 radius = self.config.saturatedColumnDilationRadius
516 with defects.bulk_update():
517 for index in range(len(satX)):
518 minX = np.clip(satX[index] - radius, 0, None)
519 maxX = np.clip(satX[index] + radius, None, exp.image.array.shape[1] - 1)
520 s = Box2I(minimum=Point2I(minX, satY[index]),
521 maximum=Point2I(maxX, satY[index]))
522 defects.append(s)
524 return defects
526 def maskBadColumns(self, exp, defects):
527 """Mask full amplifier columns if they are sufficiently bad.
529 Parameters
530 ----------
531 defects : `lsst.ip.isr.Defects`
532 The defects found in the image so far
534 Returns
535 -------
536 exp : `lsst.afw.image.exposure.Exposure`
537 The exposure in which to find defects.
538 defects : `lsst.ip.isr.Defects`
539 If the number of bad pixels in a column is not larger or
540 equal than self.config.badPixelColumnThreshold, the input
541 list is returned. Otherwise, the defects list returned
542 will include boxes that mask blocks of on-and-of pixels.
543 badColumnCount : `int`
544 Number of bad columns masked.
545 """
546 # Render the defects into an image.
547 defectImage = afwImage.ImageI(exp.getBBox())
549 for defect in defects:
550 defectImage[defect.getBBox()] = 1
552 badColumnCount = 0
554 if self.config.badPixelsToFillColumnThreshold > 0:
555 with defects.bulk_update():
556 for amp in exp.getDetector():
557 subImage = defectImage[amp.getBBox()].array
558 nInCol = np.sum(subImage, axis=0)
560 badColIndices, = (nInCol >= self.config.badPixelsToFillColumnThreshold).nonzero()
561 badColumns = badColIndices + amp.getBBox().getMinX()
563 for badColumn in badColumns:
564 s = Box2I(minimum=Point2I(badColumn, amp.getBBox().getMinY()),
565 maximum=Point2I(badColumn, amp.getBBox().getMaxY()))
566 defects.append(s)
568 badColumnCount += len(badColIndices)
570 if self.config.saturatedPixelsToFillColumnThreshold > 0:
571 mask = afwImage.Mask.getPlaneBitMask(self.config.saturatedColumnMask)
573 with defects.bulk_update():
574 for amp in exp.getDetector():
575 subMask = exp.mask[amp.getBBox()].array
576 # Turn all the SAT bits into 1s
577 subMask &= mask
578 subMask[subMask > 0] = 1
580 nInCol = np.sum(subMask, axis=0)
582 badColIndices, = (nInCol >= self.config.saturatedPixelsToFillColumnThreshold).nonzero()
583 badColumns = badColIndices + amp.getBBox().getMinX()
585 for badColumn in badColumns:
586 s = Box2I(minimum=Point2I(badColumn, amp.getBBox().getMinY()),
587 maximum=Point2I(badColumn, amp.getBBox().getMaxY()))
588 defects.append(s)
590 badColumnCount += len(badColIndices)
592 return defects, badColumnCount
594 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
595 """Mask blocks in a column if number of on-and-off bad pixels is above
596 threshold.
598 This function is called if the number of on-and-off bad pixels
599 in a column is larger or equal than
600 self.config.badOnAndOffPixelColumnThreshold.
602 Parameters
603 ---------
604 x : `list`
605 Lower left x coordinate of defect box. x coordinate is
606 along the short axis if amp.
607 y : `list`
608 Lower left y coordinate of defect box. x coordinate is
609 along the long axis if amp.
610 multipleX : list
611 List of x coordinates in amp. with multiple bad pixels
612 (i.e., columns with defects).
613 defects : `lsst.ip.isr.Defects`
614 The defcts found in the image so far
616 Returns
617 -------
618 defects : `lsst.ip.isr.Defects`
619 The defects list returned that will include boxes that
620 mask blocks of on-and-of pixels.
621 """
622 with defects.bulk_update():
623 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
624 for x0 in multipleX:
625 index = np.where(x == x0)
626 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
627 multipleY.sort() # Ensure that the y values are sorted to look for gaps.
628 minY, maxY = np.min(multipleY), np.max(multipleY)
629 # Next few lines: don't mask pixels in column if gap
630 # of good pixels between two consecutive bad pixels is
631 # larger or equal than 'goodPixelColumnGapThreshold'.
632 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
633 if len(diffIndex) != 0:
634 limits = [minY] # put the minimum first
635 for gapIndex in diffIndex:
636 limits.append(multipleY[gapIndex])
637 limits.append(multipleY[gapIndex+1])
638 limits.append(maxY) # maximum last
639 for i in np.arange(0, len(limits)-1, 2):
640 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
641 defects.append(s)
642 else: # No gap is large enough
643 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
644 defects.append(s)
645 return defects
647 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover
648 """Plot the defects found by the task.
650 Parameters
651 ----------
652 stepname : `str`
653 Debug frame to request.
654 ampImage : `lsst.afw.image.MaskedImage`
655 Amplifier image to display.
656 defects : `lsst.ip.isr.Defects`
657 The defects to plot.
658 detector : `lsst.afw.cameraGeom.Detector`
659 Detector holding camera geometry.
660 """
661 frame = getDebugFrame(self._display, stepname)
662 if frame:
663 disp = afwDisplay.Display(frame=frame)
664 disp.scale('asinh', 'zscale')
665 disp.setMaskTransparency(80)
666 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
668 maskedIm = ampImage.clone()
669 defects.maskPixels(maskedIm, "BAD")
671 mpDict = maskedIm.mask.getMaskPlaneDict()
672 for plane in mpDict.keys():
673 if plane in ['BAD']:
674 continue
675 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
677 disp.setImageColormap('gray')
678 disp.mtv(maskedIm)
679 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
680 prompt = "Press Enter to continue [c]... "
681 while True:
682 ans = input(prompt).lower()
683 if ans in ('', 'c', ):
684 break
686 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp):
687 """Make a histogram of the distribution of pixel values for
688 each amp.
690 The main image data histogram is plotted in blue. Edge
691 pixels, if masked, are in red. Note that masked edge pixels
692 do not contribute to the underflow and overflow numbers.
694 Note that this currently only supports the 16-amp LSST
695 detectors.
697 Parameters
698 ----------
699 stepname : `str`
700 Debug frame to request.
701 ampImage : `lsst.afw.image.MaskedImage`
702 Amplifier image to display.
703 nSigmaUsed : `float`
704 The number of sigma used for detection
705 exp : `lsst.afw.image.exposure.Exposure`
706 The exposure in which the defects were found.
707 """
708 frame = getDebugFrame(self._display, stepname)
709 if frame:
710 import matplotlib.pyplot as plt
712 detector = exp.getDetector()
713 nX = np.floor(np.sqrt(len(detector)))
714 nY = len(detector) // nX
715 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10))
717 expTime = exp.getInfo().getVisitInfo().getExposureTime()
719 for (amp, a) in zip(reversed(detector), ax.flatten()):
720 mi = exp.maskedImage[amp.getBBox()]
722 # normalize by expTime as we plot in ADU/s and don't
723 # always work with master calibs
724 mi.image.array /= expTime
725 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
726 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
727 # Get array of pixels
728 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
729 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
730 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
732 thrUpper = mean + nSigmaUsed*sigma
733 thrLower = mean - nSigmaUsed*sigma
735 nRight = len(imgData[imgData > thrUpper])
736 nLeft = len(imgData[imgData < thrLower])
738 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
739 leftEdge = mean - nsig * nSigmaUsed*sigma
740 rightEdge = mean + nsig * nSigmaUsed*sigma
741 nbins = np.linspace(leftEdge, rightEdge, 1000)
742 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins,
743 lw=1, edgecolor='red')
744 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins,
745 lw=3, edgecolor='blue')
747 # Report number of entries in over- and under-flow
748 # bins, i.e. off the edges of the histogram
749 nOverflow = len(imgData[imgData > rightEdge])
750 nUnderflow = len(imgData[imgData < leftEdge])
752 # Put v-lines and textboxes in
753 a.axvline(thrUpper, c='k')
754 a.axvline(thrLower, c='k')
755 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
756 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
757 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
758 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
760 # set axis limits and scales
761 a.set_ylim([1., 1.7*np.max(y)])
762 lPlot, rPlot = a.get_xlim()
763 a.set_xlim(np.array([lPlot, rPlot]))
764 a.set_yscale('log')
765 a.set_xlabel("ADU/s")
766 fig.show()
767 prompt = "Press Enter or c to continue [chp]..."
768 while True:
769 ans = input(prompt).lower()
770 if ans in ("", " ", "c",):
771 break
772 elif ans in ("p", ):
773 import pdb
774 pdb.set_trace()
775 elif ans in ("h", ):
776 print("[h]elp [c]ontinue [p]db")
777 plt.close()
780class MeasureDefectsCombinedConnections(pipeBase.PipelineTaskConnections,
781 dimensions=("instrument", "detector")):
782 inputExp = cT.Input(
783 name="dark",
784 doc="Input ISR-processed combined exposure to measure.",
785 storageClass="ExposureF",
786 dimensions=("instrument", "detector"),
787 multiple=False,
788 isCalibration=True,
789 )
790 camera = cT.PrerequisiteInput(
791 name='camera',
792 doc="Camera associated with this exposure.",
793 storageClass="Camera",
794 dimensions=("instrument", ),
795 isCalibration=True,
796 )
798 outputDefects = cT.Output(
799 name="cpPartialDefectsFromDarkCombined",
800 doc="Output measured defects.",
801 storageClass="Defects",
802 dimensions=("instrument", "detector"),
803 )
806class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig,
807 pipelineConnections=MeasureDefectsCombinedConnections):
808 """Configuration for measuring defects from combined exposures.
809 """
810 pass
813class MeasureDefectsCombinedTask(MeasureDefectsTask):
814 """Task to measure defects in combined images."""
816 ConfigClass = MeasureDefectsCombinedTaskConfig
817 _DefaultName = "cpDefectMeasureCombined"
820class MeasureDefectsCombinedWithFilterConnections(pipeBase.PipelineTaskConnections,
821 dimensions=("instrument", "detector", "physical_filter")):
822 """Task to measure defects in combined flats under a certain filter."""
823 inputExp = cT.Input(
824 name="flat",
825 doc="Input ISR-processed combined exposure to measure.",
826 storageClass="ExposureF",
827 dimensions=("instrument", "detector", "physical_filter"),
828 multiple=False,
829 isCalibration=True,
830 )
831 camera = cT.PrerequisiteInput(
832 name='camera',
833 doc="Camera associated with this exposure.",
834 storageClass="Camera",
835 dimensions=("instrument", ),
836 isCalibration=True,
837 )
839 outputDefects = cT.Output(
840 name="cpPartialDefectsFromFlatCombinedWithFilter",
841 doc="Output measured defects.",
842 storageClass="Defects",
843 dimensions=("instrument", "detector", "physical_filter"),
844 )
847class MeasureDefectsCombinedWithFilterTaskConfig(
848 MeasureDefectsTaskConfig,
849 pipelineConnections=MeasureDefectsCombinedWithFilterConnections):
850 """Configuration for measuring defects from combined exposures.
851 """
852 pass
855class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask):
856 """Task to measure defects in combined images."""
858 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig
859 _DefaultName = "cpDefectMeasureWithFilterCombined"
862class MergeDefectsConnections(pipeBase.PipelineTaskConnections,
863 dimensions=("instrument", "detector")):
864 inputDefects = cT.Input(
865 name="singleExpDefects",
866 doc="Measured defect lists.",
867 storageClass="Defects",
868 dimensions=("instrument", "detector", "exposure",),
869 multiple=True,
870 )
871 camera = cT.PrerequisiteInput(
872 name='camera',
873 doc="Camera associated with these defects.",
874 storageClass="Camera",
875 dimensions=("instrument", ),
876 isCalibration=True,
877 )
879 mergedDefects = cT.Output(
880 name="defects",
881 doc="Final merged defects.",
882 storageClass="Defects",
883 dimensions=("instrument", "detector"),
884 multiple=False,
885 isCalibration=True,
886 )
889class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig,
890 pipelineConnections=MergeDefectsConnections):
891 """Configuration for merging single exposure defects.
892 """
894 assertSameRun = pexConfig.Field(
895 dtype=bool,
896 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or "
897 "if the run key isn't found."),
898 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
899 )
900 ignoreFilters = pexConfig.Field(
901 dtype=bool,
902 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
903 " images. Allows mixing of filters in the input flats. Set to False if you think"
904 " your defects might be chromatic and want to have registry support for varying"
905 " defects with respect to filter."),
906 default=True,
907 )
908 nullFilterName = pexConfig.Field(
909 dtype=str,
910 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
911 default="NONE",
912 )
913 combinationMode = pexConfig.ChoiceField(
914 doc="Which types of defects to identify",
915 dtype=str,
916 default="FRACTION",
917 allowed={
918 "AND": "Logical AND the pixels found in each visit to form set ",
919 "OR": "Logical OR the pixels found in each visit to form set ",
920 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
921 }
922 )
923 combinationFraction = pexConfig.RangeField(
924 dtype=float,
925 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
926 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
927 " mode AND to require pixel to appear in all images."),
928 default=0.7,
929 min=0,
930 max=1,
931 )
932 nPixBorderUpDown = pexConfig.Field(
933 dtype=int,
934 doc="Number of pixels on top & bottom of image to mask as defects if edgesAsDefects is True.",
935 default=5,
936 )
937 nPixBorderLeftRight = pexConfig.Field(
938 dtype=int,
939 doc="Number of pixels on left & right of image to mask as defects if edgesAsDefects is True.",
940 default=5,
941 )
942 edgesAsDefects = pexConfig.Field(
943 dtype=bool,
944 doc="Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects.",
945 default=False,
946 )
949class MergeDefectsTask(pipeBase.PipelineTask):
950 """Merge the defects from multiple exposures.
951 """
953 ConfigClass = MergeDefectsTaskConfig
954 _DefaultName = 'cpDefectMerge'
956 def run(self, inputDefects, camera):
957 """Merge a list of single defects to find the common defect regions.
959 Parameters
960 ----------
961 inputDefects : `list` [`lsst.ip.isr.Defects`]
962 Partial defects from a single exposure.
963 camera : `lsst.afw.cameraGeom.Camera`
964 Camera to use for metadata.
966 Returns
967 -------
968 results : `lsst.pipe.base.Struct`
969 Results struct containing:
971 ``mergedDefects``
972 The defects merged from the input lists
973 (`lsst.ip.isr.Defects`).
974 """
975 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
976 if detectorId is None:
977 raise RuntimeError("Cannot identify detector id.")
978 detector = camera[detectorId]
980 imageTypes = set()
981 for inDefect in inputDefects:
982 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN')
983 imageTypes.add(imageType)
985 # Determine common defect pixels separately for each input image type.
986 splitDefects = list()
987 for imageType in imageTypes:
988 sumImage = afwImage.MaskedImageF(detector.getBBox())
989 count = 0
990 for inDefect in inputDefects:
991 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'):
992 count += 1
993 for defect in inDefect:
994 sumImage.image[defect.getBBox()] += 1.0
995 sumImage /= count
996 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
997 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType))
999 if self.config.combinationMode == 'AND':
1000 threshold = 1.0
1001 elif self.config.combinationMode == 'OR':
1002 threshold = 0.0
1003 elif self.config.combinationMode == 'FRACTION':
1004 threshold = self.config.combinationFraction
1005 else:
1006 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}")
1007 indices = np.where(sumImage.getImage().getArray() > threshold)
1008 BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
1009 sumImage.getMask().getArray()[indices] |= BADBIT
1010 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType))
1011 partialDefect = Defects.fromMask(sumImage, 'BAD')
1012 splitDefects.append(partialDefect)
1014 # Do final combination of separate image types
1015 finalImage = afwImage.MaskedImageF(detector.getBBox())
1016 for inDefect in splitDefects:
1017 for defect in inDefect:
1018 finalImage.image[defect.getBBox()] += 1
1019 finalImage /= len(splitDefects)
1020 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
1021 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, ))
1023 # This combination is the OR of all image types
1024 threshold = 0.0
1025 indices = np.where(finalImage.getImage().getArray() > threshold)
1026 BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
1027 finalImage.getMask().getArray()[indices] |= BADBIT
1028 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), ))
1030 if self.config.edgesAsDefects:
1031 self.log.info("Masking edge pixels as defects.")
1032 # This code follows the pattern from isrTask.maskEdges().
1033 if self.config.nPixBorderLeftRight > 0:
1034 box = detector.getBBox()
1035 subImage = finalImage[box]
1036 box.grow(Extent2I(-self.config.nPixBorderLeftRight, 0))
1037 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
1038 if self.config.nPixBorderUpDown > 0:
1039 box = detector.getBBox()
1040 subImage = finalImage[box]
1041 box.grow(Extent2I(0, -self.config.nPixBorderUpDown))
1042 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
1044 merged = Defects.fromMask(finalImage, 'BAD')
1045 merged.updateMetadataFromExposures(inputDefects)
1046 merged.updateMetadata(camera=camera, detector=detector, filterName=None,
1047 setCalibId=True, setDate=True)
1049 return pipeBase.Struct(
1050 mergedDefects=merged,
1051 )
1053# Subclass the MergeDefects task to reduce the input dimensions
1054# from ("instrument", "detector", "exposure") to
1055# ("instrument", "detector").
1058class MergeDefectsCombinedConnections(pipeBase.PipelineTaskConnections,
1059 dimensions=("instrument", "detector")):
1060 inputDarkDefects = cT.Input(
1061 name="cpPartialDefectsFromDarkCombined",
1062 doc="Measured defect lists.",
1063 storageClass="Defects",
1064 dimensions=("instrument", "detector",),
1065 multiple=True,
1066 )
1067 inputBiasDefects = cT.Input(
1068 name="cpPartialDefectsFromBiasCombined",
1069 doc="Additional measured defect lists.",
1070 storageClass="Defects",
1071 dimensions=("instrument", "detector",),
1072 multiple=True,
1073 )
1074 inputFlatDefects = cT.Input(
1075 name="cpPartialDefectsFromFlatCombinedWithFilter",
1076 doc="Additional measured defect lists.",
1077 storageClass="Defects",
1078 dimensions=("instrument", "detector", "physical_filter"),
1079 multiple=True,
1080 )
1081 camera = cT.PrerequisiteInput(
1082 name='camera',
1083 doc="Camera associated with these defects.",
1084 storageClass="Camera",
1085 dimensions=("instrument", ),
1086 isCalibration=True,
1087 )
1089 mergedDefects = cT.Output(
1090 name="defects",
1091 doc="Final merged defects.",
1092 storageClass="Defects",
1093 dimensions=("instrument", "detector"),
1094 multiple=False,
1095 isCalibration=True,
1096 )
1099class MergeDefectsCombinedTaskConfig(MergeDefectsTaskConfig,
1100 pipelineConnections=MergeDefectsCombinedConnections):
1101 """Configuration for merging defects from combined exposure.
1102 """
1103 def validate(self):
1104 super().validate()
1105 if self.combinationMode != 'OR':
1106 raise ValueError("combinationMode must be 'OR'")
1109class MergeDefectsCombinedTask(MergeDefectsTask):
1110 """Task to measure defects in combined images."""
1112 ConfigClass = MergeDefectsCombinedTaskConfig
1113 _DefaultName = "cpDefectMergeCombined"
1115 @staticmethod
1116 def chooseBest(inputs):
1117 """Select the input with the most exposures used."""
1118 best = 0
1119 if len(inputs) > 1:
1120 nInput = 0
1121 for num, exp in enumerate(inputs):
1122 # This technically overcounts by a factor of 3.
1123 N = len([k for k, v in exp.getMetadata().toDict().items() if "CPP_INPUT_" in k])
1124 if N > nInput:
1125 best = num
1126 nInput = N
1127 return inputs[best]
1129 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1130 inputs = butlerQC.get(inputRefs)
1131 # Turn inputFlatDefects and inputDarkDefects into a list which
1132 # is what MergeDefectsTask expects. If there are multiple,
1133 # use the one with the most inputs.
1134 tempList = [self.chooseBest(inputs['inputFlatDefects']),
1135 self.chooseBest(inputs['inputDarkDefects']),
1136 self.chooseBest(inputs['inputBiasDefects'])]
1138 # Rename inputDefects
1139 inputsCombined = {'inputDefects': tempList, 'camera': inputs['camera']}
1141 outputs = super().run(**inputsCombined)
1142 butlerQC.put(outputs, outputRefs)