Coverage for python/lsst/cp/pipe/defects.py: 20%
339 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-25 04:39 -0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-25 04:39 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask',
24 'MergeDefectsTaskConfig', 'MergeDefectsTask',
25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask',
26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ]
28import numpy as np
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as cT
33from lsstDebug import getDebugFrame
34import lsst.pex.config as pexConfig
36import lsst.afw.image as afwImage
37import lsst.afw.math as afwMath
38import lsst.afw.detection as afwDetection
39import lsst.afw.display as afwDisplay
40from lsst.afw import cameraGeom
41from lsst.geom import Box2I, Point2I
42from lsst.meas.algorithms import SourceDetectionTask
43from lsst.ip.isr import Defects, countMaskedPixels
44from lsst.pex.exceptions import InvalidParameterError
46from ._lookupStaticCalibration import lookupStaticCalibration
49class MeasureDefectsConnections(pipeBase.PipelineTaskConnections,
50 dimensions=("instrument", "exposure", "detector")):
51 inputExp = cT.Input(
52 name="defectExps",
53 doc="Input ISR-processed exposures to measure.",
54 storageClass="Exposure",
55 dimensions=("instrument", "detector", "exposure"),
56 multiple=False
57 )
58 camera = cT.PrerequisiteInput(
59 name='camera',
60 doc="Camera associated with this exposure.",
61 storageClass="Camera",
62 dimensions=("instrument", ),
63 isCalibration=True,
64 lookupFunction=lookupStaticCalibration,
65 )
67 outputDefects = cT.Output(
68 name="singleExpDefects",
69 doc="Output measured defects.",
70 storageClass="Defects",
71 dimensions=("instrument", "detector", "exposure"),
72 )
75class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig,
76 pipelineConnections=MeasureDefectsConnections):
77 """Configuration for measuring defects from a list of exposures
78 """
80 thresholdType = pexConfig.ChoiceField(
81 dtype=str,
82 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found "
83 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found "
84 "in flats, and hot pixels in darks."),
85 default='STDEV',
86 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.",
87 'VALUE': "Use pixel value to determine detection threshold."},
88 )
89 darkCurrentThreshold = pexConfig.Field(
90 dtype=float,
91 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define "
92 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."),
93 default=5,
94 )
95 fracThresholdFlat = pexConfig.Field(
96 dtype=float,
97 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark "
98 "pixels in flat images (fraction of the mean value per amplifier)."
99 "Unused if thresholdType==``STDEV``."),
100 default=0.8,
101 )
102 nSigmaBright = pexConfig.Field(
103 dtype=float,
104 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot "
105 "pixel detection. The default value was found to be "
106 "appropriate for some LSST sensors in DM-17490. "
107 "Unused if thresholdType==``VALUE``"),
108 default=4.8,
109 )
110 nSigmaDark = pexConfig.Field(
111 dtype=float,
112 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel "
113 "detection. The default value was found to be "
114 "appropriate for some LSST sensors in DM-17490. "
115 "Unused if thresholdType==``VALUE``"),
116 default=-5.0,
117 )
118 nPixBorderUpDown = pexConfig.Field(
119 dtype=int,
120 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
121 default=7,
122 )
123 nPixBorderLeftRight = pexConfig.Field(
124 dtype=int,
125 doc="Number of pixels to exclude from left & right of image when looking for defects.",
126 default=7,
127 )
128 badOnAndOffPixelColumnThreshold = pexConfig.Field(
129 dtype=int,
130 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) "
131 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the "
132 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long "
133 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all "
134 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive "
135 "non-bad pixels in BPC, an exception to the above is made and those consecutive "
136 "'goodPixelColumnGapThreshold' are not marked as bad."),
137 default=50,
138 )
139 goodPixelColumnGapThreshold = pexConfig.Field(
140 dtype=int,
141 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see "
142 "'badOnAndOffPixelColumnThreshold')."),
143 default=30,
144 )
146 def validate(self):
147 super().validate()
148 if self.nSigmaBright < 0.0:
149 raise ValueError("nSigmaBright must be above 0.0.")
150 if self.nSigmaDark > 0.0:
151 raise ValueError("nSigmaDark must be below 0.0.")
154class MeasureDefectsTask(pipeBase.PipelineTask):
155 """Measure the defects from one exposure.
156 """
158 ConfigClass = MeasureDefectsTaskConfig
159 _DefaultName = 'cpDefectMeasure'
161 def run(self, inputExp, camera):
162 """Measure one exposure for defects.
164 Parameters
165 ----------
166 inputExp : `lsst.afw.image.Exposure`
167 Exposure to examine.
168 camera : `lsst.afw.cameraGeom.Camera`
169 Camera to use for metadata.
171 Returns
172 -------
173 results : `lsst.pipe.base.Struct`
174 Results struct containing:
176 ``outputDefects``
177 The defects measured from this exposure
178 (`lsst.ip.isr.Defects`).
179 """
180 detector = inputExp.getDetector()
181 try:
182 filterName = inputExp.getFilter().physicalLabel
183 except AttributeError:
184 filterName = None
186 defects = self._findHotAndColdPixels(inputExp)
188 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN')
189 msg = "Found %s defects containing %s pixels in %s"
190 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
192 defects.updateMetadataFromExposures([inputExp])
193 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName,
194 setCalibId=True, setDate=True,
195 cpDefectGenImageType=datasetType)
197 return pipeBase.Struct(
198 outputDefects=defects,
199 )
201 @staticmethod
202 def _nPixFromDefects(defects):
203 """Count pixels in a defect.
205 Parameters
206 ----------
207 defects : `lsst.ip.isr.Defects`
208 Defects to measure.
210 Returns
211 -------
212 nPix : `int`
213 Number of defect pixels.
214 """
215 nPix = 0
216 for defect in defects:
217 nPix += defect.getBBox().getArea()
218 return nPix
220 def _findHotAndColdPixels(self, exp):
221 """Find hot and cold pixels in an image.
223 Using config-defined thresholds on a per-amp basis, mask
224 pixels that are nSigma above threshold in dark frames (hot
225 pixels), or nSigma away from the clipped mean in flats (hot &
226 cold pixels).
228 Parameters
229 ----------
230 exp : `lsst.afw.image.exposure.Exposure`
231 The exposure in which to find defects.
233 Returns
234 -------
235 defects : `lsst.ip.isr.Defects`
236 The defects found in the image.
237 """
238 self._setEdgeBits(exp)
239 maskedIm = exp.maskedImage
241 # the detection polarity for afwDetection, True for positive,
242 # False for negative, and therefore True for darks as they only have
243 # bright pixels, and both for flats, as they have bright and dark pix
244 footprintList = []
246 for amp in exp.getDetector():
247 ampImg = maskedIm[amp.getBBox()].clone()
249 # crop ampImage depending on where the amp lies in the image
250 if self.config.nPixBorderLeftRight:
251 if ampImg.getX0() == 0:
252 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
253 else:
254 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
255 if self.config.nPixBorderUpDown:
256 if ampImg.getY0() == 0:
257 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
258 else:
259 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
261 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
262 continue
264 # Remove a background estimate
265 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
266 ampImg -= meanClip
268 # Determine thresholds
269 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue()
270 expTime = exp.getInfo().getVisitInfo().getExposureTime()
271 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN')
272 if np.isnan(expTime):
273 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second",
274 expTime, amp.getName(), datasetType)
275 expTime = 1.
276 thresholdType = self.config.thresholdType
277 if thresholdType == 'VALUE':
278 # LCA-128 and eoTest: bright/hot pixels in dark images are
279 # defined as any pixel with more than 5 e-/s of dark current.
280 # We scale by the exposure time.
281 if datasetType.lower() == 'dark':
282 # hot pixel threshold
283 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain()
284 else:
285 # LCA-128 and eoTest: dark/cold pixels in flat images as
286 # defined as any pixel with photoresponse <80% of
287 # the mean (at 500nm).
289 # We subtracted the mean above, so the threshold will be
290 # negative cold pixel threshold.
291 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip
292 # Find equivalent sigma values.
293 nSigmaList = [valueThreshold/stDev]
294 else:
295 hotPixelThreshold = self.config.nSigmaBright
296 coldPixelThreshold = self.config.nSigmaDark
297 if datasetType.lower() == 'dark':
298 nSigmaList = [hotPixelThreshold]
299 valueThreshold = stDev*hotPixelThreshold
300 else:
301 nSigmaList = [hotPixelThreshold, coldPixelThreshold]
302 valueThreshold = [x*stDev for x in nSigmaList]
304 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel"
305 "Values (hot and cold pixels thresholds): %s, %s",
306 datasetType, amp.getName(), thresholdType, nSigmaList, valueThreshold)
307 mergedSet = None
308 for sigma in nSigmaList:
309 nSig = np.abs(sigma)
310 self.debugHistogram('ampFlux', ampImg, nSig, exp)
311 polarity = {-1: False, 1: True}[np.sign(sigma)]
313 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
315 try:
316 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
317 except InvalidParameterError:
318 # This occurs if the image sigma value is 0.0.
319 # Let's mask the whole area.
320 minValue = np.nanmin(ampImg.image.array) - 1.0
321 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True)
322 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
324 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
326 if mergedSet is None:
327 mergedSet = footprintSet
328 else:
329 mergedSet.merge(footprintSet)
331 footprintList += mergedSet.getFootprints()
333 self.debugView('defectMap', ampImg,
334 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector())
336 defects = Defects.fromFootprintList(footprintList)
337 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
339 return defects
341 @staticmethod
342 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
343 """Return the number of non-bad pixels in the image."""
344 nPixels = maskedIm.mask.array.size
345 nBad = countMaskedPixels(maskedIm, badMaskString)
346 return nPixels - nBad
348 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
349 """Set edge bits on an exposure or maskedImage.
351 Raises
352 ------
353 TypeError
354 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
355 """
356 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
357 mi = exposureOrMaskedImage.maskedImage
358 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
359 mi = exposureOrMaskedImage
360 else:
361 t = type(exposureOrMaskedImage)
362 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
364 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
365 if self.config.nPixBorderLeftRight:
366 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
367 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
368 if self.config.nPixBorderUpDown:
369 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
370 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
372 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
373 """Mask blocks in a column if there are on-and-off bad pixels
375 If there's a column with on and off bad pixels, mask all the
376 pixels in between, except if there is a large enough gap of
377 consecutive good pixels between two bad pixels in the column.
379 Parameters
380 ----------
381 defects : `lsst.ip.isr.Defects`
382 The defects found in the image so far
384 Returns
385 -------
386 defects : `lsst.ip.isr.Defects`
387 If the number of bad pixels in a column is not larger or
388 equal than self.config.badPixelColumnThreshold, the input
389 list is returned. Otherwise, the defects list returned
390 will include boxes that mask blocks of on-and-of pixels.
391 """
392 # Get the (x, y) values of each bad pixel in amp.
393 coordinates = []
394 for defect in defects:
395 bbox = defect.getBBox()
396 x0, y0 = bbox.getMinX(), bbox.getMinY()
397 deltaX0, deltaY0 = bbox.getDimensions()
398 for j in np.arange(y0, y0+deltaY0):
399 for i in np.arange(x0, x0 + deltaX0):
400 coordinates.append((i, j))
402 x, y = [], []
403 for coordinatePair in coordinates:
404 x.append(coordinatePair[0])
405 y.append(coordinatePair[1])
407 x = np.array(x)
408 y = np.array(y)
409 # Find the defects with same "x" (vertical) coordinate (column).
410 unique, counts = np.unique(x, return_counts=True)
411 multipleX = []
412 for (a, b) in zip(unique, counts):
413 if b >= self.config.badOnAndOffPixelColumnThreshold:
414 multipleX.append(a)
415 if len(multipleX) != 0:
416 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
418 return defects
420 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
421 """Mask blocks in a column if number of on-and-off bad pixels is above
422 threshold.
424 This function is called if the number of on-and-off bad pixels
425 in a column is larger or equal than
426 self.config.badOnAndOffPixelColumnThreshold.
428 Parameters
429 ---------
430 x : `list`
431 Lower left x coordinate of defect box. x coordinate is
432 along the short axis if amp.
433 y : `list`
434 Lower left y coordinate of defect box. x coordinate is
435 along the long axis if amp.
436 multipleX : list
437 List of x coordinates in amp. with multiple bad pixels
438 (i.e., columns with defects).
439 defects : `lsst.ip.isr.Defects`
440 The defcts found in the image so far
442 Returns
443 -------
444 defects : `lsst.ip.isr.Defects`
445 The defects list returned that will include boxes that
446 mask blocks of on-and-of pixels.
447 """
448 with defects.bulk_update():
449 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
450 for x0 in multipleX:
451 index = np.where(x == x0)
452 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
453 multipleY.sort() # Ensure that the y values are sorted to look for gaps.
454 minY, maxY = np.min(multipleY), np.max(multipleY)
455 # Next few lines: don't mask pixels in column if gap
456 # of good pixels between two consecutive bad pixels is
457 # larger or equal than 'goodPixelColumnGapThreshold'.
458 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
459 if len(diffIndex) != 0:
460 limits = [minY] # put the minimum first
461 for gapIndex in diffIndex:
462 limits.append(multipleY[gapIndex])
463 limits.append(multipleY[gapIndex+1])
464 limits.append(maxY) # maximum last
465 for i in np.arange(0, len(limits)-1, 2):
466 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
467 defects.append(s)
468 else: # No gap is large enough
469 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
470 defects.append(s)
471 return defects
473 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover
474 """Plot the defects found by the task.
476 Parameters
477 ----------
478 stepname : `str`
479 Debug frame to request.
480 ampImage : `lsst.afw.image.MaskedImage`
481 Amplifier image to display.
482 defects : `lsst.ip.isr.Defects`
483 The defects to plot.
484 detector : `lsst.afw.cameraGeom.Detector`
485 Detector holding camera geometry.
486 """
487 frame = getDebugFrame(self._display, stepname)
488 if frame:
489 disp = afwDisplay.Display(frame=frame)
490 disp.scale('asinh', 'zscale')
491 disp.setMaskTransparency(80)
492 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
494 maskedIm = ampImage.clone()
495 defects.maskPixels(maskedIm, "BAD")
497 mpDict = maskedIm.mask.getMaskPlaneDict()
498 for plane in mpDict.keys():
499 if plane in ['BAD']:
500 continue
501 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
503 disp.setImageColormap('gray')
504 disp.mtv(maskedIm)
505 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
506 prompt = "Press Enter to continue [c]... "
507 while True:
508 ans = input(prompt).lower()
509 if ans in ('', 'c', ):
510 break
512 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp):
513 """Make a histogram of the distribution of pixel values for
514 each amp.
516 The main image data histogram is plotted in blue. Edge
517 pixels, if masked, are in red. Note that masked edge pixels
518 do not contribute to the underflow and overflow numbers.
520 Note that this currently only supports the 16-amp LSST
521 detectors.
523 Parameters
524 ----------
525 stepname : `str`
526 Debug frame to request.
527 ampImage : `lsst.afw.image.MaskedImage`
528 Amplifier image to display.
529 nSigmaUsed : `float`
530 The number of sigma used for detection
531 exp : `lsst.afw.image.exposure.Exposure`
532 The exposure in which the defects were found.
533 """
534 frame = getDebugFrame(self._display, stepname)
535 if frame:
536 import matplotlib.pyplot as plt
538 detector = exp.getDetector()
539 nX = np.floor(np.sqrt(len(detector)))
540 nY = len(detector) // nX
541 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10))
543 expTime = exp.getInfo().getVisitInfo().getExposureTime()
545 for (amp, a) in zip(reversed(detector), ax.flatten()):
546 mi = exp.maskedImage[amp.getBBox()]
548 # normalize by expTime as we plot in ADU/s and don't
549 # always work with master calibs
550 mi.image.array /= expTime
551 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
552 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
553 # Get array of pixels
554 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
555 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
556 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
558 thrUpper = mean + nSigmaUsed*sigma
559 thrLower = mean - nSigmaUsed*sigma
561 nRight = len(imgData[imgData > thrUpper])
562 nLeft = len(imgData[imgData < thrLower])
564 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
565 leftEdge = mean - nsig * nSigmaUsed*sigma
566 rightEdge = mean + nsig * nSigmaUsed*sigma
567 nbins = np.linspace(leftEdge, rightEdge, 1000)
568 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins,
569 lw=1, edgecolor='red')
570 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins,
571 lw=3, edgecolor='blue')
573 # Report number of entries in over- and under-flow
574 # bins, i.e. off the edges of the histogram
575 nOverflow = len(imgData[imgData > rightEdge])
576 nUnderflow = len(imgData[imgData < leftEdge])
578 # Put v-lines and textboxes in
579 a.axvline(thrUpper, c='k')
580 a.axvline(thrLower, c='k')
581 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
582 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
583 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
584 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
586 # set axis limits and scales
587 a.set_ylim([1., 1.7*np.max(y)])
588 lPlot, rPlot = a.get_xlim()
589 a.set_xlim(np.array([lPlot, rPlot]))
590 a.set_yscale('log')
591 a.set_xlabel("ADU/s")
592 fig.show()
593 prompt = "Press Enter or c to continue [chp]..."
594 while True:
595 ans = input(prompt).lower()
596 if ans in ("", " ", "c",):
597 break
598 elif ans in ("p", ):
599 import pdb
600 pdb.set_trace()
601 elif ans in ("h", ):
602 print("[h]elp [c]ontinue [p]db")
603 plt.close()
606class MeasureDefectsCombinedConnections(MeasureDefectsConnections,
607 dimensions=("instrument", "detector")):
608 inputExp = cT.Input(
609 name="dark",
610 doc="Input ISR-processed combined exposure to measure.",
611 storageClass="Exposure",
612 dimensions=("instrument", "detector"),
613 multiple=False,
614 isCalibration=True,
615 )
616 camera = cT.PrerequisiteInput(
617 name='camera',
618 doc="Camera associated with this exposure.",
619 storageClass="Camera",
620 dimensions=("instrument", ),
621 isCalibration=True,
622 lookupFunction=lookupStaticCalibration,
623 )
625 outputDefects = cT.Output(
626 name="cpPartialDefectsFromDarkCombined",
627 doc="Output measured defects.",
628 storageClass="Defects",
629 dimensions=("instrument", "detector"),
630 )
633class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig,
634 pipelineConnections=MeasureDefectsCombinedConnections):
635 """Configuration for measuring defects from combined exposures.
636 """
637 pass
640class MeasureDefectsCombinedTask(MeasureDefectsTask):
641 """Task to measure defects in combined images."""
643 ConfigClass = MeasureDefectsCombinedTaskConfig
644 _DefaultName = "cpDefectMeasureCombined"
647class MeasureDefectsCombinedWithFilterConnections(MeasureDefectsCombinedConnections,
648 dimensions=("instrument", "detector")):
649 """Task to measure defects in combined flats under a certain filter."""
650 inputExp = cT.Input(
651 name="flat",
652 doc="Input ISR-processed combined exposure to measure.",
653 storageClass="Exposure",
654 dimensions=("instrument", "detector", "physical_filter"),
655 multiple=False,
656 isCalibration=True,
657 )
658 camera = cT.PrerequisiteInput(
659 name='camera',
660 doc="Camera associated with this exposure.",
661 storageClass="Camera",
662 dimensions=("instrument", ),
663 isCalibration=True,
664 lookupFunction=lookupStaticCalibration,
665 )
667 outputDefects = cT.Output(
668 name="cpPartialDefectsFromFlatCombinedWithFilter",
669 doc="Output measured defects.",
670 storageClass="Defects",
671 dimensions=("instrument", "detector", "physical_filter"),
672 )
675class MeasureDefectsCombinedWithFilterTaskConfig(
676 MeasureDefectsTaskConfig,
677 pipelineConnections=MeasureDefectsCombinedWithFilterConnections):
678 """Configuration for measuring defects from combined exposures.
679 """
680 pass
683class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask):
684 """Task to measure defects in combined images."""
686 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig
687 _DefaultName = "cpDefectMeasureWithFilterCombined"
690class MergeDefectsConnections(pipeBase.PipelineTaskConnections,
691 dimensions=("instrument", "detector")):
692 inputDefects = cT.Input(
693 name="singleExpDefects",
694 doc="Measured defect lists.",
695 storageClass="Defects",
696 dimensions=("instrument", "detector", "exposure",),
697 multiple=True,
698 )
699 camera = cT.PrerequisiteInput(
700 name='camera',
701 doc="Camera associated with these defects.",
702 storageClass="Camera",
703 dimensions=("instrument", ),
704 isCalibration=True,
705 lookupFunction=lookupStaticCalibration,
706 )
708 mergedDefects = cT.Output(
709 name="defects",
710 doc="Final merged defects.",
711 storageClass="Defects",
712 dimensions=("instrument", "detector"),
713 multiple=False,
714 isCalibration=True,
715 )
718class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig,
719 pipelineConnections=MergeDefectsConnections):
720 """Configuration for merging single exposure defects.
721 """
723 assertSameRun = pexConfig.Field(
724 dtype=bool,
725 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or "
726 "if the run key isn't found."),
727 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
728 )
729 ignoreFilters = pexConfig.Field(
730 dtype=bool,
731 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
732 " images. Allows mixing of filters in the input flats. Set to False if you think"
733 " your defects might be chromatic and want to have registry support for varying"
734 " defects with respect to filter."),
735 default=True,
736 )
737 nullFilterName = pexConfig.Field(
738 dtype=str,
739 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
740 default="NONE",
741 )
742 combinationMode = pexConfig.ChoiceField(
743 doc="Which types of defects to identify",
744 dtype=str,
745 default="FRACTION",
746 allowed={
747 "AND": "Logical AND the pixels found in each visit to form set ",
748 "OR": "Logical OR the pixels found in each visit to form set ",
749 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
750 }
751 )
752 combinationFraction = pexConfig.RangeField(
753 dtype=float,
754 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
755 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
756 " mode AND to require pixel to appear in all images."),
757 default=0.7,
758 min=0,
759 max=1,
760 )
761 edgesAsDefects = pexConfig.Field(
762 dtype=bool,
763 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
764 " Normal treatment is to simply exclude this region from the defect finding, such that no"
765 " defect will be located there."),
766 default=False,
767 )
770class MergeDefectsTask(pipeBase.PipelineTask):
771 """Merge the defects from multiple exposures.
772 """
774 ConfigClass = MergeDefectsTaskConfig
775 _DefaultName = 'cpDefectMerge'
777 def run(self, inputDefects, camera):
778 """Merge a list of single defects to find the common defect regions.
780 Parameters
781 ----------
782 inputDefects : `list` [`lsst.ip.isr.Defects`]
783 Partial defects from a single exposure.
784 camera : `lsst.afw.cameraGeom.Camera`
785 Camera to use for metadata.
787 Returns
788 -------
789 results : `lsst.pipe.base.Struct`
790 Results struct containing:
792 ``mergedDefects``
793 The defects merged from the input lists
794 (`lsst.ip.isr.Defects`).
795 """
796 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
797 if detectorId is None:
798 raise RuntimeError("Cannot identify detector id.")
799 detector = camera[detectorId]
801 imageTypes = set()
802 for inDefect in inputDefects:
803 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN')
804 imageTypes.add(imageType)
806 # Determine common defect pixels separately for each input image type.
807 splitDefects = list()
808 for imageType in imageTypes:
809 sumImage = afwImage.MaskedImageF(detector.getBBox())
810 count = 0
811 for inDefect in inputDefects:
812 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'):
813 count += 1
814 for defect in inDefect:
815 sumImage.image[defect.getBBox()] += 1.0
816 sumImage /= count
817 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
818 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType))
820 if self.config.combinationMode == 'AND':
821 threshold = 1.0
822 elif self.config.combinationMode == 'OR':
823 threshold = 0.0
824 elif self.config.combinationMode == 'FRACTION':
825 threshold = self.config.combinationFraction
826 else:
827 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}")
828 indices = np.where(sumImage.getImage().getArray() > threshold)
829 BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
830 sumImage.getMask().getArray()[indices] |= BADBIT
831 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType))
832 partialDefect = Defects.fromMask(sumImage, 'BAD')
833 splitDefects.append(partialDefect)
835 # Do final combination of separate image types
836 finalImage = afwImage.MaskedImageF(detector.getBBox())
837 for inDefect in splitDefects:
838 for defect in inDefect:
839 finalImage.image[defect.getBBox()] += 1
840 finalImage /= len(splitDefects)
841 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
842 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, ))
844 # This combination is the OR of all image types
845 threshold = 0.0
846 indices = np.where(finalImage.getImage().getArray() > threshold)
847 BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
848 finalImage.getMask().getArray()[indices] |= BADBIT
849 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), ))
851 if self.config.edgesAsDefects:
852 self.log.info("Masking edge pixels as defects.")
853 # Do the same as IsrTask.maskEdges()
854 box = detector.getBBox()
855 subImage = finalImage[box]
856 box.grow(-self.nPixBorder)
857 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
859 merged = Defects.fromMask(finalImage, 'BAD')
860 merged.updateMetadataFromExposures(inputDefects)
861 merged.updateMetadata(camera=camera, detector=detector, filterName=None,
862 setCalibId=True, setDate=True)
864 return pipeBase.Struct(
865 mergedDefects=merged,
866 )
868# Subclass the MergeDefects task to reduce the input dimensions
869# from ("instrument", "detector", "exposure") to
870# ("instrument", "detector").
873class MergeDefectsCombinedConnections(pipeBase.PipelineTaskConnections,
874 dimensions=("instrument", "detector")):
875 inputFlatDefects = cT.Input(
876 name="cpPartialDefectsFromDarkCombined",
877 doc="Measured defect lists.",
878 storageClass="Defects",
879 dimensions=("instrument", "detector",),
880 multiple=False,
881 )
882 inputDarkDefects = cT.Input(
883 name="cpPartialDefectsFromFlatCombinedWithFilter",
884 doc="Additional measured defect lists.",
885 storageClass="Defects",
886 dimensions=("instrument", "detector", "physical_filter"),
887 multiple=False,
888 )
889 camera = cT.PrerequisiteInput(
890 name='camera',
891 doc="Camera associated with these defects.",
892 storageClass="Camera",
893 dimensions=("instrument", ),
894 isCalibration=True,
895 lookupFunction=lookupStaticCalibration,
896 )
898 mergedDefects = cT.Output(
899 name="defectsCombined",
900 doc="Final merged defects.",
901 storageClass="Defects",
902 dimensions=("instrument", "detector"),
903 multiple=False,
904 isCalibration=True,
905 )
908class MergeDefectsCombinedTaskConfig(MergeDefectsTaskConfig,
909 pipelineConnections=MergeDefectsCombinedConnections):
910 """Configuration for merging defects from combined exposure.
911 """
912 def validate(self):
913 super().validate()
914 if self.combinationMode != 'OR':
915 raise ValueError("combinationMode must be 'OR'")
918class MergeDefectsCombinedTask(MergeDefectsTask):
919 """Task to measure defects in combined images."""
921 ConfigClass = MergeDefectsCombinedTaskConfig
922 _DefaultName = "cpDefectMergeCombined"
924 def runQuantum(self, butlerQC, inputRefs, outputRefs):
925 inputs = butlerQC.get(inputRefs)
926 # Turn inputFlatDefects and inputDarkDefects into a list
927 # which is what MergeDefectsTask expects.
928 tempList = [inputs['inputFlatDefects'], inputs['inputDarkDefects']]
929 # Rename inputDefects
930 inputsCombined = {'inputDefects': tempList, 'camera': inputs['camera']}
932 outputs = super().run(**inputsCombined)
933 butlerQC.put(outputs, outputRefs)