Coverage for python/lsst/cp/pipe/defects.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.pipe.base as pipeBase
25import lsst.pipe.base.connectionTypes as cT
27from lsstDebug import getDebugFrame
28import lsst.pex.config as pexConfig
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.detection as afwDetection
33import lsst.afw.display as afwDisplay
34from lsst.afw import cameraGeom
35from lsst.geom import Box2I, Point2I
36from lsst.meas.algorithms import SourceDetectionTask
37from lsst.ip.isr import IsrTask, Defects
38from .utils import countMaskedPixels
39from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
41from ._lookupStaticCalibration import lookupStaticCalibration
43__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask',
44 'MergeDefectsTaskConfig', 'MergeDefectsTask',
45 'FindDefectsTask', 'FindDefectsTaskConfig', ]
48class MeasureDefectsConnections(pipeBase.PipelineTaskConnections,
49 dimensions=("instrument", "exposure", "detector")):
50 inputExp = cT.Input(
51 name="defectExps",
52 doc="Input ISR-processed exposures to measure.",
53 storageClass="Exposure",
54 dimensions=("instrument", "detector", "exposure"),
55 multiple=False
56 )
57 camera = cT.PrerequisiteInput(
58 name='camera',
59 doc="Camera associated with this exposure.",
60 storageClass="Camera",
61 dimensions=("instrument", ),
62 isCalibration=True,
63 lookupFunction=lookupStaticCalibration,
64 )
66 outputDefects = cT.Output(
67 name="singleExpDefects",
68 doc="Output measured defects.",
69 storageClass="Defects",
70 dimensions=("instrument", "detector", "exposure"),
71 )
74class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig,
75 pipelineConnections=MeasureDefectsConnections):
76 """Configuration for measuring defects from a list of exposures
77 """
78 nSigmaBright = pexConfig.Field(
79 dtype=float,
80 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be",
81 " appropriate for some LSST sensors in DM-17490."),
82 default=4.8,
83 )
84 nSigmaDark = pexConfig.Field(
85 dtype=float,
86 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be",
87 " appropriate for some LSST sensors in DM-17490."),
88 default=-5.0,
89 )
90 nPixBorderUpDown = pexConfig.Field(
91 dtype=int,
92 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
93 default=7,
94 )
95 nPixBorderLeftRight = pexConfig.Field(
96 dtype=int,
97 doc="Number of pixels to exclude from left & right of image when looking for defects.",
98 default=7,
99 )
100 badOnAndOffPixelColumnThreshold = pexConfig.Field(
101 dtype=int,
102 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ",
103 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ",
104 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ",
105 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ",
106 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ",
107 "non-bad pixels in BPC, an exception to the above is made and those consecutive ",
108 "'goodPixelColumnGapThreshold' are not marked as bad."),
109 default=50,
110 )
111 goodPixelColumnGapThreshold = pexConfig.Field(
112 dtype=int,
113 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ",
114 "'badOnAndOffPixelColumnThreshold')."),
115 default=30,
116 )
118 def validate(self):
119 super().validate()
120 if self.nSigmaBright < 0.0:
121 raise ValueError("nSigmaBright must be above 0.0.")
122 if self.nSigmaDark > 0.0:
123 raise ValueError("nSigmaDark must be below 0.0.")
126class MeasureDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
127 """Measure the defects from one exposure.
128 """
129 ConfigClass = MeasureDefectsTaskConfig
130 _DefaultName = 'cpDefectMeasure'
132 def run(self, inputExp, camera):
133 """Measure one exposure for defects.
135 Parameters
136 ----------
137 inputExp : `lsst.afw.image.Exposure`
138 Exposure to examine.
139 camera : `lsst.afw.cameraGeom.Camera`
140 Camera to use for metadata.
142 Returns
143 -------
144 results : `lsst.pipe.base.Struct`
145 Results struct containing:
146 - ``outputDefects` : `lsst.ip.isr.Defects`
147 The defects measured from this exposure.
148 """
149 detector = inputExp.getDetector()
151 filterName = inputExp.getFilterLabel().physicalLabel
152 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN')
154 if datasetType.lower() == 'dark':
155 nSigmaList = [self.config.nSigmaBright]
156 else:
157 nSigmaList = [self.config.nSigmaBright, self.config.nSigmaDark]
158 defects = self.findHotAndColdPixels(inputExp, nSigmaList)
160 msg = "Found %s defects containing %s pixels in %s"
161 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
163 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName,
164 setCalibId=True, setDate=True,
165 cpDefectGenImageType=datasetType)
167 return pipeBase.Struct(
168 outputDefects=defects,
169 )
171 @staticmethod
172 def _nPixFromDefects(defects):
173 """Count pixels in a defect.
175 Parameters
176 ----------
177 defects : `lsst.ip.isr.Defects`
178 Defects to measure.
180 Returns
181 -------
182 nPix : `int`
183 Number of defect pixels.
184 """
185 nPix = 0
186 for defect in defects:
187 nPix += defect.getBBox().getArea()
188 return nPix
190 def findHotAndColdPixels(self, exp, nSigma):
191 """Find hot and cold pixels in an image.
193 Using config-defined thresholds on a per-amp basis, mask
194 pixels that are nSigma above threshold in dark frames (hot
195 pixels), or nSigma away from the clipped mean in flats (hot &
196 cold pixels).
198 Parameters
199 ----------
200 exp : `lsst.afw.image.exposure.Exposure`
201 The exposure in which to find defects.
202 nSigma : `list [ `float` ]
203 Detection threshold to use. Positive for DETECTED pixels,
204 negative for DETECTED_NEGATIVE pixels.
206 Returns
207 -------
208 defects : `lsst.ip.isr.Defect`
209 The defects found in the image.
211 """
213 self._setEdgeBits(exp)
214 maskedIm = exp.maskedImage
216 # the detection polarity for afwDetection, True for positive,
217 # False for negative, and therefore True for darks as they only have
218 # bright pixels, and both for flats, as they have bright and dark pix
219 footprintList = []
221 for amp in exp.getDetector():
222 ampImg = maskedIm[amp.getBBox()].clone()
224 # crop ampImage depending on where the amp lies in the image
225 if self.config.nPixBorderLeftRight:
226 if ampImg.getX0() == 0:
227 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
228 else:
229 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
230 if self.config.nPixBorderUpDown:
231 if ampImg.getY0() == 0:
232 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
233 else:
234 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
236 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
237 continue
239 # Remove a background estimate
240 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
242 mergedSet = None
243 for sigma in nSigma:
244 nSig = np.abs(sigma)
245 self.debugHistogram('ampFlux', ampImg, nSig, exp)
246 polarity = {-1: False, 1: True}[np.sign(sigma)]
248 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
250 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
251 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
253 if mergedSet is None:
254 mergedSet = footprintSet
255 else:
256 mergedSet.merge(footprintSet)
258 footprintList += mergedSet.getFootprints()
260 self.debugView('defectMap', ampImg,
261 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector())
263 defects = Defects.fromFootprintList(footprintList)
264 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
266 return defects
268 @staticmethod
269 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
270 """Return the number of non-bad pixels in the image."""
271 nPixels = maskedIm.mask.array.size
272 nBad = countMaskedPixels(maskedIm, badMaskString)
273 return nPixels - nBad
275 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
276 """Set edge bits on an exposure or maskedImage.
277 Raises
278 ------
279 TypeError
280 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
281 """
282 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
283 mi = exposureOrMaskedImage.maskedImage
284 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
285 mi = exposureOrMaskedImage
286 else:
287 t = type(exposureOrMaskedImage)
288 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
290 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
291 if self.config.nPixBorderLeftRight:
292 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
293 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
294 if self.config.nPixBorderUpDown:
295 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
296 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
298 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
299 """Mask blocks in a column if there are on-and-off bad pixels
301 If there's a column with on and off bad pixels, mask all the
302 pixels in between, except if there is a large enough gap of
303 consecutive good pixels between two bad pixels in the column.
305 Parameters
306 ---------
307 defects: `lsst.ip.isr.Defect`
308 The defects found in the image so far
310 Returns
311 ------
312 defects: `lsst.ip.isr.Defect`
313 If the number of bad pixels in a column is not larger or
314 equal than self.config.badPixelColumnThreshold, the iput
315 list is returned. Otherwise, the defects list returned
316 will include boxes that mask blocks of on-and-of pixels.
318 """
319 # Get the (x, y) values of each bad pixel in amp.
320 coordinates = []
321 for defect in defects:
322 bbox = defect.getBBox()
323 x0, y0 = bbox.getMinX(), bbox.getMinY()
324 deltaX0, deltaY0 = bbox.getDimensions()
325 for j in np.arange(y0, y0+deltaY0):
326 for i in np.arange(x0, x0 + deltaX0):
327 coordinates.append((i, j))
329 x, y = [], []
330 for coordinatePair in coordinates:
331 x.append(coordinatePair[0])
332 y.append(coordinatePair[1])
334 x = np.array(x)
335 y = np.array(y)
336 # Find the defects with same "x" (vertical) coordinate (column).
337 unique, counts = np.unique(x, return_counts=True)
338 multipleX = []
339 for (a, b) in zip(unique, counts):
340 if b >= self.config.badOnAndOffPixelColumnThreshold:
341 multipleX.append(a)
342 if len(multipleX) != 0:
343 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
345 return defects
347 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
348 """Mask blocks in a column if number of on-and-off bad pixels is above threshold.
350 This function is called if the number of on-and-off bad pixels
351 in a column is larger or equal than
352 self.config.badOnAndOffPixelColumnThreshold.
354 Parameters
355 ---------
356 x: `list`
357 Lower left x coordinate of defect box. x coordinate is
358 along the short axis if amp.
359 y: `list`
360 Lower left y coordinate of defect box. x coordinate is
361 along the long axis if amp.
362 multipleX: list
363 List of x coordinates in amp. with multiple bad pixels
364 (i.e., columns with defects).
365 defects: `lsst.ip.isr.Defect`
366 The defcts found in the image so far
368 Returns
369 -------
370 defects: `lsst.ip.isr.Defect`
371 The defects list returned that will include boxes that
372 mask blocks of on-and-of pixels.
374 """
375 with defects.bulk_update():
376 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
377 for x0 in multipleX:
378 index = np.where(x == x0)
379 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
380 minY, maxY = np.min(multipleY), np.max(multipleY)
381 # Next few lines: don't mask pixels in column if gap of good pixels between
382 # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'.
383 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
384 if len(diffIndex) != 0:
385 limits = [minY] # put the minimum first
386 for gapIndex in diffIndex:
387 limits.append(multipleY[gapIndex])
388 limits.append(multipleY[gapIndex+1])
389 limits.append(maxY) # maximum last
390 assert len(limits)%2 == 0, 'limits is even by design, but check anyways'
391 for i in np.arange(0, len(limits)-1, 2):
392 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
393 defects.append(s)
394 else: # No gap is large enough
395 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
396 defects.append(s)
397 return defects
399 def debugView(self, stepname, ampImage, defects, detector):
400 # def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover
401 """Plot the defects found by the task.
403 Parameters
404 ----------
405 exp : `lsst.afw.image.exposure.Exposure`
406 The exposure in which the defects were found.
407 visit : `int`
408 The visit number.
409 defects : `lsst.ip.isr.Defect`
410 The defects to plot.
411 imageType : `str`
412 The type of image, either 'dark' or 'flat'.
413 """
414 frame = getDebugFrame(self._display, stepname)
415 if frame:
416 disp = afwDisplay.Display(frame=frame)
417 disp.scale('asinh', 'zscale')
418 disp.setMaskTransparency(80)
419 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
421 maskedIm = ampImage.clone()
422 defects.maskPixels(maskedIm, "BAD")
424 mpDict = maskedIm.mask.getMaskPlaneDict()
425 for plane in mpDict.keys():
426 if plane in ['BAD']:
427 continue
428 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
430 disp.setImageColormap('gray')
431 disp.mtv(maskedIm)
432 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
433 prompt = "Press Enter to continue [c]... "
434 while True:
435 ans = input(prompt).lower()
436 if ans in ('', 'c', ):
437 break
439 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp):
440 """
441 Make a histogram of the distribution of pixel values for each amp.
443 The main image data histogram is plotted in blue. Edge pixels,
444 if masked, are in red. Note that masked edge pixels do not contribute
445 to the underflow and overflow numbers.
447 Note that this currently only supports the 16-amp LSST detectors.
449 Parameters
450 ----------
451 dataRef : `lsst.daf.persistence.ButlerDataRef`
452 dataRef for the detector.
453 exp : `lsst.afw.image.exposure.Exposure`
454 The exposure in which the defects were found.
455 visit : `int`
456 The visit number.
457 nSigmaUsed : `float`
458 The number of sigma used for detection
459 """
460 frame = getDebugFrame(self._display, stepname)
461 if frame:
462 import matplotlib.pyplot as plt
464 detector = exp.getDetector()
465 nX = np.floor(np.sqrt(len(detector)))
466 nY = len(detector) // nX
467 fig, ax = plt.subplots(nrows=nY, ncols=nX, sharex='col', sharey='row', figsize=(13, 10))
469 expTime = exp.getInfo().getVisitInfo().getExposureTime()
471 for (amp, a) in zip(reversed(detector), ax.flatten()):
472 mi = exp.maskedImage[amp.getBBox()]
474 # normalize by expTime as we plot in ADU/s and don't always work with master calibs
475 mi.image.array /= expTime
476 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
477 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
478 # Get array of pixels
479 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
480 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
481 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
483 thrUpper = mean + nSigmaUsed*sigma
484 thrLower = mean - nSigmaUsed*sigma
486 nRight = len(imgData[imgData > thrUpper])
487 nLeft = len(imgData[imgData < thrLower])
489 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
490 leftEdge = mean - nsig * nSigmaUsed*sigma
491 rightEdge = mean + nsig * nSigmaUsed*sigma
492 nbins = np.linspace(leftEdge, rightEdge, 1000)
493 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins,
494 lw=1, edgecolor='red')
495 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins,
496 lw=3, edgecolor='blue')
498 # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram
499 nOverflow = len(imgData[imgData > rightEdge])
500 nUnderflow = len(imgData[imgData < leftEdge])
502 # Put v-lines and textboxes in
503 a.axvline(thrUpper, c='k')
504 a.axvline(thrLower, c='k')
505 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
506 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
507 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
508 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
510 # set axis limits and scales
511 a.set_ylim([1., 1.7*np.max(y)])
512 lPlot, rPlot = a.get_xlim()
513 a.set_xlim(np.array([lPlot, rPlot]))
514 a.set_yscale('log')
515 a.set_xlabel("ADU/s")
516 return
519class MergeDefectsConnections(pipeBase.PipelineTaskConnections,
520 dimensions=("instrument", "detector")):
521 inputDefects = cT.Input(
522 name="singleExpDefects",
523 doc="Measured defect lists.",
524 storageClass="Defects",
525 dimensions=("instrument", "detector", "exposure"),
526 multiple=True,
527 )
528 camera = cT.PrerequisiteInput(
529 name='camera',
530 doc="Camera associated with these defects.",
531 storageClass="Camera",
532 dimensions=("instrument", ),
533 isCalibration=True,
534 lookupFunction=lookupStaticCalibration,
535 )
537 mergedDefects = cT.Output(
538 name="defects",
539 doc="Final merged defects.",
540 storageClass="Defects",
541 dimensions=("instrument", "detector"),
542 multiple=False,
543 isCalibration=True,
544 )
547class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig,
548 pipelineConnections=MergeDefectsConnections):
549 """Configuration for merging single exposure defects.
550 """
551 assertSameRun = pexConfig.Field(
552 dtype=bool,
553 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or"
554 "if the run key isn't found."),
555 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
556 )
557 ignoreFilters = pexConfig.Field(
558 dtype=bool,
559 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
560 " images. Allows mixing of filters in the input flats. Set to False if you think"
561 " your defects might be chromatic and want to have registry support for varying"
562 " defects with respect to filter."),
563 default=True,
564 )
565 nullFilterName = pexConfig.Field(
566 dtype=str,
567 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
568 default="NONE",
569 )
570 combinationMode = pexConfig.ChoiceField(
571 doc="Which types of defects to identify",
572 dtype=str,
573 default="FRACTION",
574 allowed={
575 "AND": "Logical AND the pixels found in each visit to form set ",
576 "OR": "Logical OR the pixels found in each visit to form set ",
577 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
578 }
579 )
580 combinationFraction = pexConfig.RangeField(
581 dtype=float,
582 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
583 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
584 " mode AND to require pixel to appear in all images."),
585 default=0.7,
586 min=0,
587 max=1,
588 )
589 edgesAsDefects = pexConfig.Field(
590 dtype=bool,
591 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
592 " Normal treatment is to simply exclude this region from the defect finding, such that no"
593 " defect will be located there."),
594 default=False,
595 )
598class MergeDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
599 """Merge the defects from multiple exposures.
600 """
601 ConfigClass = MergeDefectsTaskConfig
602 _DefaultName = 'cpDefectMerge'
604 def run(self, inputDefects, camera):
605 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
606 if detectorId is None:
607 raise RuntimeError("Cannot identify detector id.")
608 detector = camera[detectorId]
610 imageTypes = set()
611 for inDefect in inputDefects:
612 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN')
613 imageTypes.add(imageType)
615 # Determine common defect pixels separately for each input image type.
616 splitDefects = list()
617 for imageType in imageTypes:
618 sumImage = afwImage.MaskedImageF(detector.getBBox())
619 count = 0
620 for inDefect in inputDefects:
621 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'):
622 count += 1
623 for defect in inDefect:
624 sumImage.image[defect.getBBox()] += 1.0
625 sumImage /= count
626 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
627 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType))
629 if self.config.combinationMode == 'AND':
630 threshold = 1.0
631 elif self.config.combinationMode == 'OR':
632 threshold = 0.0
633 elif self.config.combinationMode == 'FRACTION':
634 threshold = self.config.combinationFraction
635 else:
636 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}")
637 indices = np.where(sumImage.getImage().getArray() > threshold)
638 BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
639 sumImage.getMask().getArray()[indices] |= BADBIT
640 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType))
641 partialDefect = Defects.fromMask(sumImage, 'BAD')
642 splitDefects.append(partialDefect)
644 # Do final combination of separate image types
645 finalImage = afwImage.MaskedImageF(detector.getBBox())
646 for inDefect in splitDefects:
647 for defect in inDefect:
648 finalImage.image[defect.getBBox()] += 1
649 finalImage /= len(splitDefects)
650 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
651 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, ))
653 # This combination is the OR of all image types
654 threshold = 0.0
655 indices = np.where(finalImage.getImage().getArray() > threshold)
656 BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
657 finalImage.getMask().getArray()[indices] |= BADBIT
658 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), ))
660 if self.config.edgesAsDefects:
661 self.log.info("Masking edge pixels as defects.")
662 # Do the same as IsrTask.maskEdges()
663 box = detector.getBBox()
664 subImage = finalImage[box]
665 box.grow(-self.nPixBorder)
666 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
668 merged = Defects.fromMask(finalImage, 'BAD')
669 merged.updateMetadata(camera=camera, detector=detector, filterName=None,
670 setCalibId=True, setDate=True)
672 return pipeBase.Struct(
673 mergedDefects=merged,
674 )
677class FindDefectsTaskConfig(pexConfig.Config):
678 measure = pexConfig.ConfigurableField(
679 target=MeasureDefectsTask,
680 doc="Task to measure single frame defects.",
681 )
682 merge = pexConfig.ConfigurableField(
683 target=MergeDefectsTask,
684 doc="Task to merge multiple defects together.",
685 )
687 isrForFlats = pexConfig.ConfigurableField(
688 target=IsrTask,
689 doc="Task to perform instrumental signature removal",
690 )
691 isrForDarks = pexConfig.ConfigurableField(
692 target=IsrTask,
693 doc="Task to perform instrumental signature removal",
694 )
695 isrMandatoryStepsFlats = pexConfig.ListField(
696 dtype=str,
697 doc=("isr operations that must be performed for valid results when using flats."
698 " Raises if any of these are False"),
699 default=['doAssembleCcd', 'doFringe']
700 )
701 isrMandatoryStepsDarks = pexConfig.ListField(
702 dtype=str,
703 doc=("isr operations that must be performed for valid results when using darks. "
704 "Raises if any of these are False"),
705 default=['doAssembleCcd', 'doFringe']
706 )
707 isrForbiddenStepsFlats = pexConfig.ListField(
708 dtype=str,
709 doc=("isr operations that must NOT be performed for valid results when using flats."
710 " Raises if any of these are True"),
711 default=['doBrighterFatter', 'doUseOpticsTransmission',
712 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
713 )
714 isrForbiddenStepsDarks = pexConfig.ListField(
715 dtype=str,
716 doc=("isr operations that must NOT be performed for valid results when using darks."
717 " Raises if any of these are True"),
718 default=['doBrighterFatter', 'doUseOpticsTransmission',
719 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
720 )
721 isrDesirableSteps = pexConfig.ListField(
722 dtype=str,
723 doc=("isr operations that it is advisable to perform, but are not mission-critical."
724 " WARNs are logged for any of these found to be False."),
725 default=['doBias']
726 )
728 ccdKey = pexConfig.Field(
729 dtype=str,
730 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
731 default='ccd',
732 )
733 imageTypeKey = pexConfig.Field(
734 dtype=str,
735 doc="The key for the butler to use by which to check whether images are darks or flats",
736 default='imageType',
737 )
740class FindDefectsTask(pipeBase.CmdLineTask):
741 """Task for finding defects in sensors.
743 The task has two modes of operation, defect finding in raws and in
744 master calibrations, which work as follows.
746 Master calib defect finding
747 ----------------------------
749 A single visit number is supplied, for which the corresponding flat & dark
750 will be used. This is because, at present at least, there is no way to pass
751 a calibration exposure ID from the command line to a command line task.
753 The task retrieves the corresponding dark and flat exposures for the
754 supplied visit. If a flat is available the task will (be able to) look
755 for both bright and dark defects. If only a dark is found then only bright
756 defects will be sought.
758 All pixels above/below the specified nSigma which lie with the specified
759 borders for flats/darks are identified as defects.
761 Raw visit defect finding
762 ------------------------
764 A list of exposure IDs are supplied for defect finding. The task will
765 detect bright pixels in the dark frames, if supplied, and bright & dark
766 pixels in the flats, if supplied, i.e. if you only supply darks you will
767 only be given bright defects. This is done automatically from the imageType
768 of the exposure, so the input exposure list can be a mix.
770 As with the master calib detection, all pixels above/below the specified
771 nSigma which lie with the specified borders for flats/darks are identified
772 as defects. Then, a post-processing step is done to merge these detections,
773 with pixels appearing in a fraction [0..1] of the images are kept as defects
774 and those appearing below that occurrence-threshold are discarded.
775 """
776 ConfigClass = FindDefectsTaskConfig
777 _DefaultName = "findDefects"
779 RunnerClass = DataRefListRunner
781 def __init__(self, **kwargs):
782 super().__init__(**kwargs)
783 self.makeSubtask("measure")
784 self.makeSubtask("merge")
786 @pipeBase.timeMethod
787 def runDataRef(self, dataRefList):
788 """Run the defect finding task.
790 Find the defects, as described in the main task docstring, from a
791 dataRef and a list of visit(s).
793 Parameters
794 ----------
795 dataRefList : `list` [`lsst.daf.persistence.ButlerDataRef`]
796 dataRefs for the data to be checked for defects.
798 Returns
799 -------
800 result : `lsst.pipe.base.Struct`
801 Result struct with Components:
803 - ``defects`` : `lsst.ip.isr.Defect`
804 The defects found by the task.
805 - ``exitStatus`` : `int`
806 The exit code.
807 """
808 dataRef = dataRefList[0]
809 camera = dataRef.get("camera")
811 singleExpDefects = []
812 activeChip = None
813 for dataRef in dataRefList:
814 exposure = dataRef.get("postISRCCD")
815 if activeChip:
816 if exposure.getDetector().getName() != activeChip:
817 raise RuntimeError("Too many input detectors supplied!")
818 else:
819 activeChip = exposure.getDetector().getName()
821 result = self.measure.run(exposure, camera)
822 singleExpDefects.append(result.outputDefects)
824 finalResults = self.merge.run(singleExpDefects, camera)
825 metadata = finalResults.mergedDefects.getMetadata()
826 inputDims = {'calibDate': metadata['CALIBDATE'],
827 'raftName': metadata['RAFTNAME'],
828 'detectorName': metadata['SLOTNAME'],
829 'detector': metadata['DETECTOR'],
830 'ccd': metadata['DETECTOR'],
831 'ccdnum': metadata['DETECTOR']}
833 butler = dataRef.getButler()
834 butler.put(finalResults.mergedDefects, "defects", inputDims)
836 return finalResults