Coverage for python/lsst/cp/pipe/defects.py: 22%
313 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-03 10:00 +0000
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-03 10:00 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import numpy as np
24import lsst.pipe.base as pipeBase
25import lsst.pipe.base.connectionTypes as cT
27from lsstDebug import getDebugFrame
28import lsst.pex.config as pexConfig
30import lsst.afw.image as afwImage
31import lsst.afw.math as afwMath
32import lsst.afw.detection as afwDetection
33import lsst.afw.display as afwDisplay
34from lsst.afw import cameraGeom
35from lsst.geom import Box2I, Point2I
36from lsst.meas.algorithms import SourceDetectionTask
37from lsst.ip.isr import IsrTask, Defects
38from .utils import countMaskedPixels
39from lsst.pipe.tasks.getRepositoryData import DataRefListRunner
40from lsst.utils.timer import timeMethod
42from ._lookupStaticCalibration import lookupStaticCalibration
44__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask',
45 'MergeDefectsTaskConfig', 'MergeDefectsTask',
46 'FindDefectsTask', 'FindDefectsTaskConfig', ]
49class MeasureDefectsConnections(pipeBase.PipelineTaskConnections,
50 dimensions=("instrument", "exposure", "detector")):
51 inputExp = cT.Input(
52 name="defectExps",
53 doc="Input ISR-processed exposures to measure.",
54 storageClass="Exposure",
55 dimensions=("instrument", "detector", "exposure"),
56 multiple=False
57 )
58 camera = cT.PrerequisiteInput(
59 name='camera',
60 doc="Camera associated with this exposure.",
61 storageClass="Camera",
62 dimensions=("instrument", ),
63 isCalibration=True,
64 lookupFunction=lookupStaticCalibration,
65 )
67 outputDefects = cT.Output(
68 name="singleExpDefects",
69 doc="Output measured defects.",
70 storageClass="Defects",
71 dimensions=("instrument", "detector", "exposure"),
72 )
75class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig,
76 pipelineConnections=MeasureDefectsConnections):
77 """Configuration for measuring defects from a list of exposures
78 """
80 nSigmaBright = pexConfig.Field(
81 dtype=float,
82 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be"
83 " appropriate for some LSST sensors in DM-17490."),
84 default=4.8,
85 )
86 nSigmaDark = pexConfig.Field(
87 dtype=float,
88 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be"
89 " appropriate for some LSST sensors in DM-17490."),
90 default=-5.0,
91 )
92 nPixBorderUpDown = pexConfig.Field(
93 dtype=int,
94 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
95 default=7,
96 )
97 nPixBorderLeftRight = pexConfig.Field(
98 dtype=int,
99 doc="Number of pixels to exclude from left & right of image when looking for defects.",
100 default=7,
101 )
102 badOnAndOffPixelColumnThreshold = pexConfig.Field(
103 dtype=int,
104 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) "
105 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the "
106 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long "
107 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all "
108 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive "
109 "non-bad pixels in BPC, an exception to the above is made and those consecutive "
110 "'goodPixelColumnGapThreshold' are not marked as bad."),
111 default=50,
112 )
113 goodPixelColumnGapThreshold = pexConfig.Field(
114 dtype=int,
115 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see "
116 "'badOnAndOffPixelColumnThreshold')."),
117 default=30,
118 )
120 def validate(self):
121 super().validate()
122 if self.nSigmaBright < 0.0:
123 raise ValueError("nSigmaBright must be above 0.0.")
124 if self.nSigmaDark > 0.0:
125 raise ValueError("nSigmaDark must be below 0.0.")
128class MeasureDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
129 """Measure the defects from one exposure.
130 """
132 ConfigClass = MeasureDefectsTaskConfig
133 _DefaultName = 'cpDefectMeasure'
135 def run(self, inputExp, camera):
136 """Measure one exposure for defects.
138 Parameters
139 ----------
140 inputExp : `lsst.afw.image.Exposure`
141 Exposure to examine.
142 camera : `lsst.afw.cameraGeom.Camera`
143 Camera to use for metadata.
145 Returns
146 -------
147 results : `lsst.pipe.base.Struct`
148 Results struct containing:
150 ``outputDefects``
151 The defects measured from this exposure
152 (`lsst.ip.isr.Defects`).
153 """
154 detector = inputExp.getDetector()
156 filterName = inputExp.getFilterLabel().physicalLabel
157 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN')
159 if datasetType.lower() == 'dark':
160 nSigmaList = [self.config.nSigmaBright]
161 else:
162 nSigmaList = [self.config.nSigmaBright, self.config.nSigmaDark]
163 defects = self.findHotAndColdPixels(inputExp, nSigmaList)
165 msg = "Found %s defects containing %s pixels in %s"
166 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
168 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName,
169 setCalibId=True, setDate=True,
170 cpDefectGenImageType=datasetType)
172 return pipeBase.Struct(
173 outputDefects=defects,
174 )
176 @staticmethod
177 def _nPixFromDefects(defects):
178 """Count pixels in a defect.
180 Parameters
181 ----------
182 defects : `lsst.ip.isr.Defects`
183 Defects to measure.
185 Returns
186 -------
187 nPix : `int`
188 Number of defect pixels.
189 """
190 nPix = 0
191 for defect in defects:
192 nPix += defect.getBBox().getArea()
193 return nPix
195 def findHotAndColdPixels(self, exp, nSigma):
196 """Find hot and cold pixels in an image.
198 Using config-defined thresholds on a per-amp basis, mask
199 pixels that are nSigma above threshold in dark frames (hot
200 pixels), or nSigma away from the clipped mean in flats (hot &
201 cold pixels).
203 Parameters
204 ----------
205 exp : `lsst.afw.image.exposure.Exposure`
206 The exposure in which to find defects.
207 nSigma : `list` [`float`]
208 Detection threshold to use. Positive for DETECTED pixels,
209 negative for DETECTED_NEGATIVE pixels.
211 Returns
212 -------
213 defects : `lsst.ip.isr.Defects`
214 The defects found in the image.
215 """
216 self._setEdgeBits(exp)
217 maskedIm = exp.maskedImage
219 # the detection polarity for afwDetection, True for positive,
220 # False for negative, and therefore True for darks as they only have
221 # bright pixels, and both for flats, as they have bright and dark pix
222 footprintList = []
224 for amp in exp.getDetector():
225 ampImg = maskedIm[amp.getBBox()].clone()
227 # crop ampImage depending on where the amp lies in the image
228 if self.config.nPixBorderLeftRight:
229 if ampImg.getX0() == 0:
230 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
231 else:
232 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
233 if self.config.nPixBorderUpDown:
234 if ampImg.getY0() == 0:
235 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
236 else:
237 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
239 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
240 continue
242 # Remove a background estimate
243 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
245 mergedSet = None
246 for sigma in nSigma:
247 nSig = np.abs(sigma)
248 self.debugHistogram('ampFlux', ampImg, nSig, exp)
249 polarity = {-1: False, 1: True}[np.sign(sigma)]
251 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
253 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
254 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
256 if mergedSet is None:
257 mergedSet = footprintSet
258 else:
259 mergedSet.merge(footprintSet)
261 footprintList += mergedSet.getFootprints()
263 self.debugView('defectMap', ampImg,
264 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector())
266 defects = Defects.fromFootprintList(footprintList)
267 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
269 return defects
271 @staticmethod
272 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
273 """Return the number of non-bad pixels in the image."""
274 nPixels = maskedIm.mask.array.size
275 nBad = countMaskedPixels(maskedIm, badMaskString)
276 return nPixels - nBad
278 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
279 """Set edge bits on an exposure or maskedImage.
281 Raises
282 ------
283 TypeError
284 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
285 """
286 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
287 mi = exposureOrMaskedImage.maskedImage
288 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
289 mi = exposureOrMaskedImage
290 else:
291 t = type(exposureOrMaskedImage)
292 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
294 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
295 if self.config.nPixBorderLeftRight:
296 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
297 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
298 if self.config.nPixBorderUpDown:
299 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
300 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
302 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
303 """Mask blocks in a column if there are on-and-off bad pixels
305 If there's a column with on and off bad pixels, mask all the
306 pixels in between, except if there is a large enough gap of
307 consecutive good pixels between two bad pixels in the column.
309 Parameters
310 ----------
311 defects : `lsst.ip.isr.Defects`
312 The defects found in the image so far
314 Returns
315 -------
316 defects : `lsst.ip.isr.Defects`
317 If the number of bad pixels in a column is not larger or
318 equal than self.config.badPixelColumnThreshold, the input
319 list is returned. Otherwise, the defects list returned
320 will include boxes that mask blocks of on-and-of pixels.
321 """
322 # Get the (x, y) values of each bad pixel in amp.
323 coordinates = []
324 for defect in defects:
325 bbox = defect.getBBox()
326 x0, y0 = bbox.getMinX(), bbox.getMinY()
327 deltaX0, deltaY0 = bbox.getDimensions()
328 for j in np.arange(y0, y0+deltaY0):
329 for i in np.arange(x0, x0 + deltaX0):
330 coordinates.append((i, j))
332 x, y = [], []
333 for coordinatePair in coordinates:
334 x.append(coordinatePair[0])
335 y.append(coordinatePair[1])
337 x = np.array(x)
338 y = np.array(y)
339 # Find the defects with same "x" (vertical) coordinate (column).
340 unique, counts = np.unique(x, return_counts=True)
341 multipleX = []
342 for (a, b) in zip(unique, counts):
343 if b >= self.config.badOnAndOffPixelColumnThreshold:
344 multipleX.append(a)
345 if len(multipleX) != 0:
346 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
348 return defects
350 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
351 """Mask blocks in a column if number of on-and-off bad pixels is above
352 threshold.
354 This function is called if the number of on-and-off bad pixels
355 in a column is larger or equal than
356 self.config.badOnAndOffPixelColumnThreshold.
358 Parameters
359 ---------
360 x : `list`
361 Lower left x coordinate of defect box. x coordinate is
362 along the short axis if amp.
363 y : `list`
364 Lower left y coordinate of defect box. x coordinate is
365 along the long axis if amp.
366 multipleX : list
367 List of x coordinates in amp. with multiple bad pixels
368 (i.e., columns with defects).
369 defects : `lsst.ip.isr.Defects`
370 The defcts found in the image so far
372 Returns
373 -------
374 defects : `lsst.ip.isr.Defects`
375 The defects list returned that will include boxes that
376 mask blocks of on-and-of pixels.
377 """
378 with defects.bulk_update():
379 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
380 for x0 in multipleX:
381 index = np.where(x == x0)
382 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
383 minY, maxY = np.min(multipleY), np.max(multipleY)
384 # Next few lines: don't mask pixels in column if gap
385 # of good pixels between two consecutive bad pixels is
386 # larger or equal than 'goodPixelColumnGapThreshold'.
387 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
388 if len(diffIndex) != 0:
389 limits = [minY] # put the minimum first
390 for gapIndex in diffIndex:
391 limits.append(multipleY[gapIndex])
392 limits.append(multipleY[gapIndex+1])
393 limits.append(maxY) # maximum last
394 assert len(limits)%2 == 0, 'limits is even by design, but check anyways'
395 for i in np.arange(0, len(limits)-1, 2):
396 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
397 defects.append(s)
398 else: # No gap is large enough
399 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
400 defects.append(s)
401 return defects
403 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover
404 """Plot the defects found by the task.
406 Parameters
407 ----------
408 stepname : `str`
409 Debug frame to request.
410 ampImage : `lsst.afw.image.MaskedImage`
411 Amplifier image to display.
412 defects : `lsst.ip.isr.Defects`
413 The defects to plot.
414 detector : `lsst.afw.cameraGeom.Detector`
415 Detector holding camera geometry.
416 """
417 frame = getDebugFrame(self._display, stepname)
418 if frame:
419 disp = afwDisplay.Display(frame=frame)
420 disp.scale('asinh', 'zscale')
421 disp.setMaskTransparency(80)
422 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
424 maskedIm = ampImage.clone()
425 defects.maskPixels(maskedIm, "BAD")
427 mpDict = maskedIm.mask.getMaskPlaneDict()
428 for plane in mpDict.keys():
429 if plane in ['BAD']:
430 continue
431 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
433 disp.setImageColormap('gray')
434 disp.mtv(maskedIm)
435 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
436 prompt = "Press Enter to continue [c]... "
437 while True:
438 ans = input(prompt).lower()
439 if ans in ('', 'c', ):
440 break
442 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp):
443 """Make a histogram of the distribution of pixel values for
444 each amp.
446 The main image data histogram is plotted in blue. Edge
447 pixels, if masked, are in red. Note that masked edge pixels
448 do not contribute to the underflow and overflow numbers.
450 Note that this currently only supports the 16-amp LSST
451 detectors.
453 Parameters
454 ----------
455 stepname : `str`
456 Debug frame to request.
457 ampImage : `lsst.afw.image.MaskedImage`
458 Amplifier image to display.
459 nSigmaUsed : `float`
460 The number of sigma used for detection
461 exp : `lsst.afw.image.exposure.Exposure`
462 The exposure in which the defects were found.
463 """
464 frame = getDebugFrame(self._display, stepname)
465 if frame:
466 import matplotlib.pyplot as plt
468 detector = exp.getDetector()
469 nX = np.floor(np.sqrt(len(detector)))
470 nY = len(detector) // nX
471 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10))
473 expTime = exp.getInfo().getVisitInfo().getExposureTime()
475 for (amp, a) in zip(reversed(detector), ax.flatten()):
476 mi = exp.maskedImage[amp.getBBox()]
478 # normalize by expTime as we plot in ADU/s and don't
479 # always work with master calibs
480 mi.image.array /= expTime
481 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
482 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
483 # Get array of pixels
484 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
485 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
486 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
488 thrUpper = mean + nSigmaUsed*sigma
489 thrLower = mean - nSigmaUsed*sigma
491 nRight = len(imgData[imgData > thrUpper])
492 nLeft = len(imgData[imgData < thrLower])
494 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
495 leftEdge = mean - nsig * nSigmaUsed*sigma
496 rightEdge = mean + nsig * nSigmaUsed*sigma
497 nbins = np.linspace(leftEdge, rightEdge, 1000)
498 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins,
499 lw=1, edgecolor='red')
500 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins,
501 lw=3, edgecolor='blue')
503 # Report number of entries in over- and under-flow
504 # bins, i.e. off the edges of the histogram
505 nOverflow = len(imgData[imgData > rightEdge])
506 nUnderflow = len(imgData[imgData < leftEdge])
508 # Put v-lines and textboxes in
509 a.axvline(thrUpper, c='k')
510 a.axvline(thrLower, c='k')
511 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
512 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
513 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
514 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
516 # set axis limits and scales
517 a.set_ylim([1., 1.7*np.max(y)])
518 lPlot, rPlot = a.get_xlim()
519 a.set_xlim(np.array([lPlot, rPlot]))
520 a.set_yscale('log')
521 a.set_xlabel("ADU/s")
522 fig.show()
523 prompt = "Press Enter or c to continue [chp]..."
524 while True:
525 ans = input(prompt).lower()
526 if ans in ("", " ", "c",):
527 break
528 elif ans in ("p", ):
529 import pdb
530 pdb.set_trace()
531 elif ans in ("h", ):
532 print("[h]elp [c]ontinue [p]db")
533 plt.close()
536class MergeDefectsConnections(pipeBase.PipelineTaskConnections,
537 dimensions=("instrument", "detector")):
538 inputDefects = cT.Input(
539 name="singleExpDefects",
540 doc="Measured defect lists.",
541 storageClass="Defects",
542 dimensions=("instrument", "detector", "exposure"),
543 multiple=True,
544 )
545 camera = cT.PrerequisiteInput(
546 name='camera',
547 doc="Camera associated with these defects.",
548 storageClass="Camera",
549 dimensions=("instrument", ),
550 isCalibration=True,
551 lookupFunction=lookupStaticCalibration,
552 )
554 mergedDefects = cT.Output(
555 name="defects",
556 doc="Final merged defects.",
557 storageClass="Defects",
558 dimensions=("instrument", "detector"),
559 multiple=False,
560 isCalibration=True,
561 )
564class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig,
565 pipelineConnections=MergeDefectsConnections):
566 """Configuration for merging single exposure defects.
567 """
569 assertSameRun = pexConfig.Field(
570 dtype=bool,
571 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or "
572 "if the run key isn't found."),
573 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
574 )
575 ignoreFilters = pexConfig.Field(
576 dtype=bool,
577 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
578 " images. Allows mixing of filters in the input flats. Set to False if you think"
579 " your defects might be chromatic and want to have registry support for varying"
580 " defects with respect to filter."),
581 default=True,
582 )
583 nullFilterName = pexConfig.Field(
584 dtype=str,
585 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
586 default="NONE",
587 )
588 combinationMode = pexConfig.ChoiceField(
589 doc="Which types of defects to identify",
590 dtype=str,
591 default="FRACTION",
592 allowed={
593 "AND": "Logical AND the pixels found in each visit to form set ",
594 "OR": "Logical OR the pixels found in each visit to form set ",
595 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
596 }
597 )
598 combinationFraction = pexConfig.RangeField(
599 dtype=float,
600 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
601 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
602 " mode AND to require pixel to appear in all images."),
603 default=0.7,
604 min=0,
605 max=1,
606 )
607 edgesAsDefects = pexConfig.Field(
608 dtype=bool,
609 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
610 " Normal treatment is to simply exclude this region from the defect finding, such that no"
611 " defect will be located there."),
612 default=False,
613 )
616class MergeDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
617 """Merge the defects from multiple exposures.
618 """
620 ConfigClass = MergeDefectsTaskConfig
621 _DefaultName = 'cpDefectMerge'
623 def run(self, inputDefects, camera):
624 """Merge a list of single defects to find the common defect regions.
626 Parameters
627 ----------
628 inputDefects : `list` [`lsst.ip.isr.Defects`]
629 Partial defects from a single exposure.
630 camera : `lsst.afw.cameraGeom.Camera`
631 Camera to use for metadata.
633 Returns
634 -------
635 results : `lsst.pipe.base.Struct`
636 Results struct containing:
638 ``mergedDefects``
639 The defects merged from the input lists
640 (`lsst.ip.isr.Defects`).
641 """
642 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
643 if detectorId is None:
644 raise RuntimeError("Cannot identify detector id.")
645 detector = camera[detectorId]
647 imageTypes = set()
648 for inDefect in inputDefects:
649 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN')
650 imageTypes.add(imageType)
652 # Determine common defect pixels separately for each input image type.
653 splitDefects = list()
654 for imageType in imageTypes:
655 sumImage = afwImage.MaskedImageF(detector.getBBox())
656 count = 0
657 for inDefect in inputDefects:
658 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'):
659 count += 1
660 for defect in inDefect:
661 sumImage.image[defect.getBBox()] += 1.0
662 sumImage /= count
663 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
664 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType))
666 if self.config.combinationMode == 'AND':
667 threshold = 1.0
668 elif self.config.combinationMode == 'OR':
669 threshold = 0.0
670 elif self.config.combinationMode == 'FRACTION':
671 threshold = self.config.combinationFraction
672 else:
673 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}")
674 indices = np.where(sumImage.getImage().getArray() > threshold)
675 BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
676 sumImage.getMask().getArray()[indices] |= BADBIT
677 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType))
678 partialDefect = Defects.fromMask(sumImage, 'BAD')
679 splitDefects.append(partialDefect)
681 # Do final combination of separate image types
682 finalImage = afwImage.MaskedImageF(detector.getBBox())
683 for inDefect in splitDefects:
684 for defect in inDefect:
685 finalImage.image[defect.getBBox()] += 1
686 finalImage /= len(splitDefects)
687 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
688 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, ))
690 # This combination is the OR of all image types
691 threshold = 0.0
692 indices = np.where(finalImage.getImage().getArray() > threshold)
693 BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
694 finalImage.getMask().getArray()[indices] |= BADBIT
695 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), ))
697 if self.config.edgesAsDefects:
698 self.log.info("Masking edge pixels as defects.")
699 # Do the same as IsrTask.maskEdges()
700 box = detector.getBBox()
701 subImage = finalImage[box]
702 box.grow(-self.nPixBorder)
703 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
705 merged = Defects.fromMask(finalImage, 'BAD')
706 merged.updateMetadata(camera=camera, detector=detector, filterName=None,
707 setCalibId=True, setDate=True)
709 return pipeBase.Struct(
710 mergedDefects=merged,
711 )
714class FindDefectsTaskConfig(pexConfig.Config):
715 measure = pexConfig.ConfigurableField(
716 target=MeasureDefectsTask,
717 doc="Task to measure single frame defects.",
718 )
719 merge = pexConfig.ConfigurableField(
720 target=MergeDefectsTask,
721 doc="Task to merge multiple defects together.",
722 )
724 isrForFlats = pexConfig.ConfigurableField(
725 target=IsrTask,
726 doc="Task to perform instrumental signature removal",
727 )
728 isrForDarks = pexConfig.ConfigurableField(
729 target=IsrTask,
730 doc="Task to perform instrumental signature removal",
731 )
732 isrMandatoryStepsFlats = pexConfig.ListField(
733 dtype=str,
734 doc=("isr operations that must be performed for valid results when using flats."
735 " Raises if any of these are False"),
736 default=['doAssembleCcd', 'doFringe']
737 )
738 isrMandatoryStepsDarks = pexConfig.ListField(
739 dtype=str,
740 doc=("isr operations that must be performed for valid results when using darks. "
741 "Raises if any of these are False"),
742 default=['doAssembleCcd', 'doFringe']
743 )
744 isrForbiddenStepsFlats = pexConfig.ListField(
745 dtype=str,
746 doc=("isr operations that must NOT be performed for valid results when using flats."
747 " Raises if any of these are True"),
748 default=['doBrighterFatter', 'doUseOpticsTransmission',
749 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
750 )
751 isrForbiddenStepsDarks = pexConfig.ListField(
752 dtype=str,
753 doc=("isr operations that must NOT be performed for valid results when using darks."
754 " Raises if any of these are True"),
755 default=['doBrighterFatter', 'doUseOpticsTransmission',
756 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
757 )
758 isrDesirableSteps = pexConfig.ListField(
759 dtype=str,
760 doc=("isr operations that it is advisable to perform, but are not mission-critical."
761 " WARNs are logged for any of these found to be False."),
762 default=['doBias']
763 )
765 ccdKey = pexConfig.Field(
766 dtype=str,
767 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
768 default='ccd',
769 )
770 imageTypeKey = pexConfig.Field(
771 dtype=str,
772 doc="The key for the butler to use by which to check whether images are darks or flats",
773 default='imageType',
774 )
777class FindDefectsTask(pipeBase.CmdLineTask):
778 """Task for finding defects in sensors.
780 The task has two modes of operation, defect finding in raws and in
781 master calibrations, which work as follows.
783 **Master calib defect finding**
785 A single visit number is supplied, for which the corresponding
786 flat & dark will be used. This is because, at present at least,
787 there is no way to pass a calibration exposure ID from the command
788 line to a command line task.
790 The task retrieves the corresponding dark and flat exposures for
791 the supplied visit. If a flat is available the task will (be able
792 to) look for both bright and dark defects. If only a dark is found
793 then only bright defects will be sought.
795 All pixels above/below the specified nSigma which lie with the
796 specified borders for flats/darks are identified as defects.
798 **Raw visit defect finding**
800 A list of exposure IDs are supplied for defect finding. The task
801 will detect bright pixels in the dark frames, if supplied, and
802 bright & dark pixels in the flats, if supplied, i.e. if you only
803 supply darks you will only be given bright defects. This is done
804 automatically from the imageType of the exposure, so the input
805 exposure list can be a mix.
807 As with the master calib detection, all pixels above/below the
808 specified nSigma which lie with the specified borders for
809 flats/darks are identified as defects. Then, a post-processing
810 step is done to merge these detections, with pixels appearing in a
811 fraction [0..1] of the images are kept as defects and those
812 appearing below that occurrence-threshold are discarded.
813 """
815 ConfigClass = FindDefectsTaskConfig
816 _DefaultName = "findDefects"
818 RunnerClass = DataRefListRunner
820 def __init__(self, **kwargs):
821 super().__init__(**kwargs)
822 self.makeSubtask("measure")
823 self.makeSubtask("merge")
825 @timeMethod
826 def runDataRef(self, dataRefList):
827 """Run the defect finding task.
829 Find the defects, as described in the main task docstring, from a
830 dataRef and a list of visit(s).
832 Parameters
833 ----------
834 dataRefList : `list` [`lsst.daf.persistence.ButlerDataRef`]
835 dataRefs for the data to be checked for defects.
837 Returns
838 -------
839 result : `lsst.pipe.base.Struct`
840 Result struct with Components:
842 ``defects``
843 The defects found by the task (`lsst.ip.isr.Defects`).
844 ``exitStatus``
845 The exit code (`int`).
846 """
847 dataRef = dataRefList[0]
848 camera = dataRef.get("camera")
850 singleExpDefects = []
851 activeChip = None
852 for dataRef in dataRefList:
853 exposure = dataRef.get("postISRCCD")
854 if activeChip:
855 if exposure.getDetector().getName() != activeChip:
856 raise RuntimeError("Too many input detectors supplied!")
857 else:
858 activeChip = exposure.getDetector().getName()
860 result = self.measure.run(exposure, camera)
861 singleExpDefects.append(result.outputDefects)
863 finalResults = self.merge.run(singleExpDefects, camera)
864 metadata = finalResults.mergedDefects.getMetadata()
865 inputDims = {'calibDate': metadata['CALIBDATE'],
866 'raftName': metadata['RAFTNAME'],
867 'detectorName': metadata['SLOTNAME'],
868 'detector': metadata['DETECTOR'],
869 'ccd': metadata['DETECTOR'],
870 'ccdnum': metadata['DETECTOR']}
872 butler = dataRef.getButler()
873 butler.put(finalResults.mergedDefects, "defects", inputDims)
875 return finalResults