Coverage for python/lsst/cp/pipe/defects.py: 20%
341 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-04 09:08 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-04 09:08 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask',
24 'MergeDefectsTaskConfig', 'MergeDefectsTask',
25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask',
26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ]
28import numpy as np
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as cT
33from lsstDebug import getDebugFrame
34import lsst.pex.config as pexConfig
36import lsst.afw.image as afwImage
37import lsst.afw.math as afwMath
38import lsst.afw.detection as afwDetection
39import lsst.afw.display as afwDisplay
40from lsst.afw import cameraGeom
41from lsst.geom import Box2I, Point2I
42from lsst.meas.algorithms import SourceDetectionTask
43from lsst.ip.isr import Defects, countMaskedPixels
44from lsst.pex.exceptions import InvalidParameterError
47class MeasureDefectsConnections(pipeBase.PipelineTaskConnections,
48 dimensions=("instrument", "exposure", "detector")):
49 inputExp = cT.Input(
50 name="defectExps",
51 doc="Input ISR-processed exposures to measure.",
52 storageClass="Exposure",
53 dimensions=("instrument", "detector", "exposure"),
54 multiple=False
55 )
56 camera = cT.PrerequisiteInput(
57 name='camera',
58 doc="Camera associated with this exposure.",
59 storageClass="Camera",
60 dimensions=("instrument", ),
61 isCalibration=True,
62 )
64 outputDefects = cT.Output(
65 name="singleExpDefects",
66 doc="Output measured defects.",
67 storageClass="Defects",
68 dimensions=("instrument", "detector", "exposure"),
69 )
72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig,
73 pipelineConnections=MeasureDefectsConnections):
74 """Configuration for measuring defects from a list of exposures
75 """
77 thresholdType = pexConfig.ChoiceField(
78 dtype=str,
79 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found "
80 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found "
81 "in flats, and hot pixels in darks."),
82 default='STDEV',
83 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.",
84 'VALUE': "Use pixel value to determine detection threshold."},
85 )
86 darkCurrentThreshold = pexConfig.Field(
87 dtype=float,
88 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define "
89 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."),
90 default=5,
91 )
92 fracThresholdFlat = pexConfig.Field(
93 dtype=float,
94 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark "
95 "pixels in flat images (fraction of the mean value per amplifier)."
96 "Unused if thresholdType==``STDEV``."),
97 default=0.8,
98 )
99 nSigmaBright = pexConfig.Field(
100 dtype=float,
101 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot "
102 "pixel detection. The default value was found to be "
103 "appropriate for some LSST sensors in DM-17490. "
104 "Unused if thresholdType==``VALUE``"),
105 default=4.8,
106 )
107 nSigmaDark = pexConfig.Field(
108 dtype=float,
109 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel "
110 "detection. The default value was found to be "
111 "appropriate for some LSST sensors in DM-17490. "
112 "Unused if thresholdType==``VALUE``"),
113 default=-5.0,
114 )
115 nPixBorderUpDown = pexConfig.Field(
116 dtype=int,
117 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
118 default=7,
119 )
120 nPixBorderLeftRight = pexConfig.Field(
121 dtype=int,
122 doc="Number of pixels to exclude from left & right of image when looking for defects.",
123 default=7,
124 )
125 badOnAndOffPixelColumnThreshold = pexConfig.Field(
126 dtype=int,
127 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) "
128 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the "
129 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long "
130 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all "
131 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive "
132 "non-bad pixels in BPC, an exception to the above is made and those consecutive "
133 "'goodPixelColumnGapThreshold' are not marked as bad."),
134 default=50,
135 )
136 goodPixelColumnGapThreshold = pexConfig.Field(
137 dtype=int,
138 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see "
139 "'badOnAndOffPixelColumnThreshold')."),
140 default=30,
141 )
143 def validate(self):
144 super().validate()
145 if self.nSigmaBright < 0.0:
146 raise ValueError("nSigmaBright must be above 0.0.")
147 if self.nSigmaDark > 0.0:
148 raise ValueError("nSigmaDark must be below 0.0.")
151class MeasureDefectsTask(pipeBase.PipelineTask):
152 """Measure the defects from one exposure.
153 """
155 ConfigClass = MeasureDefectsTaskConfig
156 _DefaultName = 'cpDefectMeasure'
158 def run(self, inputExp, camera):
159 """Measure one exposure for defects.
161 Parameters
162 ----------
163 inputExp : `lsst.afw.image.Exposure`
164 Exposure to examine.
165 camera : `lsst.afw.cameraGeom.Camera`
166 Camera to use for metadata.
168 Returns
169 -------
170 results : `lsst.pipe.base.Struct`
171 Results struct containing:
173 ``outputDefects``
174 The defects measured from this exposure
175 (`lsst.ip.isr.Defects`).
176 """
177 detector = inputExp.getDetector()
178 try:
179 filterName = inputExp.getFilter().physicalLabel
180 except AttributeError:
181 filterName = None
183 defects = self._findHotAndColdPixels(inputExp)
185 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN')
186 msg = "Found %s defects containing %s pixels in %s"
187 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
189 defects.updateMetadataFromExposures([inputExp])
190 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName,
191 setCalibId=True, setDate=True,
192 cpDefectGenImageType=datasetType)
194 return pipeBase.Struct(
195 outputDefects=defects,
196 )
198 @staticmethod
199 def _nPixFromDefects(defects):
200 """Count pixels in a defect.
202 Parameters
203 ----------
204 defects : `lsst.ip.isr.Defects`
205 Defects to measure.
207 Returns
208 -------
209 nPix : `int`
210 Number of defect pixels.
211 """
212 nPix = 0
213 for defect in defects:
214 nPix += defect.getBBox().getArea()
215 return nPix
217 def _findHotAndColdPixels(self, exp):
218 """Find hot and cold pixels in an image.
220 Using config-defined thresholds on a per-amp basis, mask
221 pixels that are nSigma above threshold in dark frames (hot
222 pixels), or nSigma away from the clipped mean in flats (hot &
223 cold pixels).
225 Parameters
226 ----------
227 exp : `lsst.afw.image.exposure.Exposure`
228 The exposure in which to find defects.
230 Returns
231 -------
232 defects : `lsst.ip.isr.Defects`
233 The defects found in the image.
234 """
235 self._setEdgeBits(exp)
236 maskedIm = exp.maskedImage
238 # the detection polarity for afwDetection, True for positive,
239 # False for negative, and therefore True for darks as they only have
240 # bright pixels, and both for flats, as they have bright and dark pix
241 footprintList = []
243 for amp in exp.getDetector():
244 ampImg = maskedIm[amp.getBBox()].clone()
246 # crop ampImage depending on where the amp lies in the image
247 if self.config.nPixBorderLeftRight:
248 if ampImg.getX0() == 0:
249 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
250 else:
251 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
252 if self.config.nPixBorderUpDown:
253 if ampImg.getY0() == 0:
254 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
255 else:
256 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
258 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
259 continue
261 # Remove a background estimate
262 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
263 ampImg -= meanClip
265 # Determine thresholds
266 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue()
267 expTime = exp.getInfo().getVisitInfo().getExposureTime()
268 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN')
269 if np.isnan(expTime):
270 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second",
271 expTime, amp.getName(), datasetType)
272 expTime = 1.
273 thresholdType = self.config.thresholdType
274 if thresholdType == 'VALUE':
275 # LCA-128 and eoTest: bright/hot pixels in dark images are
276 # defined as any pixel with more than 5 e-/s of dark current.
277 # We scale by the exposure time.
278 if datasetType.lower() == 'dark':
279 # hot pixel threshold
280 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain()
281 else:
282 # LCA-128 and eoTest: dark/cold pixels in flat images as
283 # defined as any pixel with photoresponse <80% of
284 # the mean (at 500nm).
286 # We subtracted the mean above, so the threshold will be
287 # negative cold pixel threshold.
288 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip
289 # Find equivalent sigma values.
290 if stDev == 0.0:
291 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.",
292 stDev, amp.getName(), datasetType)
293 nSigmaList = [np.inf]
294 else:
295 nSigmaList = [valueThreshold/stDev]
296 else:
297 hotPixelThreshold = self.config.nSigmaBright
298 coldPixelThreshold = self.config.nSigmaDark
299 if datasetType.lower() == 'dark':
300 nSigmaList = [hotPixelThreshold]
301 valueThreshold = stDev*hotPixelThreshold
302 else:
303 nSigmaList = [hotPixelThreshold, coldPixelThreshold]
304 valueThreshold = [x*stDev for x in nSigmaList]
306 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel"
307 "Values (hot and cold pixels thresholds): %s, %s",
308 datasetType, amp.getName(), thresholdType, nSigmaList, valueThreshold)
309 mergedSet = None
310 for sigma in nSigmaList:
311 nSig = np.abs(sigma)
312 self.debugHistogram('ampFlux', ampImg, nSig, exp)
313 polarity = {-1: False, 1: True}[np.sign(sigma)]
315 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
317 try:
318 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
319 except InvalidParameterError:
320 # This occurs if the image sigma value is 0.0.
321 # Let's mask the whole area.
322 minValue = np.nanmin(ampImg.image.array) - 1.0
323 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True)
324 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
326 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
328 if mergedSet is None:
329 mergedSet = footprintSet
330 else:
331 mergedSet.merge(footprintSet)
333 footprintList += mergedSet.getFootprints()
335 self.debugView('defectMap', ampImg,
336 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector())
338 defects = Defects.fromFootprintList(footprintList)
339 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
341 return defects
343 @staticmethod
344 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
345 """Return the number of non-bad pixels in the image."""
346 nPixels = maskedIm.mask.array.size
347 nBad = countMaskedPixels(maskedIm, badMaskString)
348 return nPixels - nBad
350 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
351 """Set edge bits on an exposure or maskedImage.
353 Raises
354 ------
355 TypeError
356 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
357 """
358 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
359 mi = exposureOrMaskedImage.maskedImage
360 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
361 mi = exposureOrMaskedImage
362 else:
363 t = type(exposureOrMaskedImage)
364 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
366 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
367 if self.config.nPixBorderLeftRight:
368 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
369 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
370 if self.config.nPixBorderUpDown:
371 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
372 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
374 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
375 """Mask blocks in a column if there are on-and-off bad pixels
377 If there's a column with on and off bad pixels, mask all the
378 pixels in between, except if there is a large enough gap of
379 consecutive good pixels between two bad pixels in the column.
381 Parameters
382 ----------
383 defects : `lsst.ip.isr.Defects`
384 The defects found in the image so far
386 Returns
387 -------
388 defects : `lsst.ip.isr.Defects`
389 If the number of bad pixels in a column is not larger or
390 equal than self.config.badPixelColumnThreshold, the input
391 list is returned. Otherwise, the defects list returned
392 will include boxes that mask blocks of on-and-of pixels.
393 """
394 # Get the (x, y) values of each bad pixel in amp.
395 coordinates = []
396 for defect in defects:
397 bbox = defect.getBBox()
398 x0, y0 = bbox.getMinX(), bbox.getMinY()
399 deltaX0, deltaY0 = bbox.getDimensions()
400 for j in np.arange(y0, y0+deltaY0):
401 for i in np.arange(x0, x0 + deltaX0):
402 coordinates.append((i, j))
404 x, y = [], []
405 for coordinatePair in coordinates:
406 x.append(coordinatePair[0])
407 y.append(coordinatePair[1])
409 x = np.array(x)
410 y = np.array(y)
411 # Find the defects with same "x" (vertical) coordinate (column).
412 unique, counts = np.unique(x, return_counts=True)
413 multipleX = []
414 for (a, b) in zip(unique, counts):
415 if b >= self.config.badOnAndOffPixelColumnThreshold:
416 multipleX.append(a)
417 if len(multipleX) != 0:
418 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
420 return defects
422 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
423 """Mask blocks in a column if number of on-and-off bad pixels is above
424 threshold.
426 This function is called if the number of on-and-off bad pixels
427 in a column is larger or equal than
428 self.config.badOnAndOffPixelColumnThreshold.
430 Parameters
431 ---------
432 x : `list`
433 Lower left x coordinate of defect box. x coordinate is
434 along the short axis if amp.
435 y : `list`
436 Lower left y coordinate of defect box. x coordinate is
437 along the long axis if amp.
438 multipleX : list
439 List of x coordinates in amp. with multiple bad pixels
440 (i.e., columns with defects).
441 defects : `lsst.ip.isr.Defects`
442 The defcts found in the image so far
444 Returns
445 -------
446 defects : `lsst.ip.isr.Defects`
447 The defects list returned that will include boxes that
448 mask blocks of on-and-of pixels.
449 """
450 with defects.bulk_update():
451 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
452 for x0 in multipleX:
453 index = np.where(x == x0)
454 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
455 multipleY.sort() # Ensure that the y values are sorted to look for gaps.
456 minY, maxY = np.min(multipleY), np.max(multipleY)
457 # Next few lines: don't mask pixels in column if gap
458 # of good pixels between two consecutive bad pixels is
459 # larger or equal than 'goodPixelColumnGapThreshold'.
460 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
461 if len(diffIndex) != 0:
462 limits = [minY] # put the minimum first
463 for gapIndex in diffIndex:
464 limits.append(multipleY[gapIndex])
465 limits.append(multipleY[gapIndex+1])
466 limits.append(maxY) # maximum last
467 for i in np.arange(0, len(limits)-1, 2):
468 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
469 defects.append(s)
470 else: # No gap is large enough
471 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
472 defects.append(s)
473 return defects
475 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover
476 """Plot the defects found by the task.
478 Parameters
479 ----------
480 stepname : `str`
481 Debug frame to request.
482 ampImage : `lsst.afw.image.MaskedImage`
483 Amplifier image to display.
484 defects : `lsst.ip.isr.Defects`
485 The defects to plot.
486 detector : `lsst.afw.cameraGeom.Detector`
487 Detector holding camera geometry.
488 """
489 frame = getDebugFrame(self._display, stepname)
490 if frame:
491 disp = afwDisplay.Display(frame=frame)
492 disp.scale('asinh', 'zscale')
493 disp.setMaskTransparency(80)
494 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
496 maskedIm = ampImage.clone()
497 defects.maskPixels(maskedIm, "BAD")
499 mpDict = maskedIm.mask.getMaskPlaneDict()
500 for plane in mpDict.keys():
501 if plane in ['BAD']:
502 continue
503 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
505 disp.setImageColormap('gray')
506 disp.mtv(maskedIm)
507 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
508 prompt = "Press Enter to continue [c]... "
509 while True:
510 ans = input(prompt).lower()
511 if ans in ('', 'c', ):
512 break
514 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp):
515 """Make a histogram of the distribution of pixel values for
516 each amp.
518 The main image data histogram is plotted in blue. Edge
519 pixels, if masked, are in red. Note that masked edge pixels
520 do not contribute to the underflow and overflow numbers.
522 Note that this currently only supports the 16-amp LSST
523 detectors.
525 Parameters
526 ----------
527 stepname : `str`
528 Debug frame to request.
529 ampImage : `lsst.afw.image.MaskedImage`
530 Amplifier image to display.
531 nSigmaUsed : `float`
532 The number of sigma used for detection
533 exp : `lsst.afw.image.exposure.Exposure`
534 The exposure in which the defects were found.
535 """
536 frame = getDebugFrame(self._display, stepname)
537 if frame:
538 import matplotlib.pyplot as plt
540 detector = exp.getDetector()
541 nX = np.floor(np.sqrt(len(detector)))
542 nY = len(detector) // nX
543 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10))
545 expTime = exp.getInfo().getVisitInfo().getExposureTime()
547 for (amp, a) in zip(reversed(detector), ax.flatten()):
548 mi = exp.maskedImage[amp.getBBox()]
550 # normalize by expTime as we plot in ADU/s and don't
551 # always work with master calibs
552 mi.image.array /= expTime
553 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
554 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
555 # Get array of pixels
556 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
557 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
558 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
560 thrUpper = mean + nSigmaUsed*sigma
561 thrLower = mean - nSigmaUsed*sigma
563 nRight = len(imgData[imgData > thrUpper])
564 nLeft = len(imgData[imgData < thrLower])
566 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
567 leftEdge = mean - nsig * nSigmaUsed*sigma
568 rightEdge = mean + nsig * nSigmaUsed*sigma
569 nbins = np.linspace(leftEdge, rightEdge, 1000)
570 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins,
571 lw=1, edgecolor='red')
572 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins,
573 lw=3, edgecolor='blue')
575 # Report number of entries in over- and under-flow
576 # bins, i.e. off the edges of the histogram
577 nOverflow = len(imgData[imgData > rightEdge])
578 nUnderflow = len(imgData[imgData < leftEdge])
580 # Put v-lines and textboxes in
581 a.axvline(thrUpper, c='k')
582 a.axvline(thrLower, c='k')
583 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
584 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
585 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
586 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
588 # set axis limits and scales
589 a.set_ylim([1., 1.7*np.max(y)])
590 lPlot, rPlot = a.get_xlim()
591 a.set_xlim(np.array([lPlot, rPlot]))
592 a.set_yscale('log')
593 a.set_xlabel("ADU/s")
594 fig.show()
595 prompt = "Press Enter or c to continue [chp]..."
596 while True:
597 ans = input(prompt).lower()
598 if ans in ("", " ", "c",):
599 break
600 elif ans in ("p", ):
601 import pdb
602 pdb.set_trace()
603 elif ans in ("h", ):
604 print("[h]elp [c]ontinue [p]db")
605 plt.close()
608class MeasureDefectsCombinedConnections(MeasureDefectsConnections,
609 dimensions=("instrument", "detector")):
610 inputExp = cT.Input(
611 name="dark",
612 doc="Input ISR-processed combined exposure to measure.",
613 storageClass="ExposureF",
614 dimensions=("instrument", "detector"),
615 multiple=False,
616 isCalibration=True,
617 )
618 camera = cT.PrerequisiteInput(
619 name='camera',
620 doc="Camera associated with this exposure.",
621 storageClass="Camera",
622 dimensions=("instrument", ),
623 isCalibration=True,
624 )
626 outputDefects = cT.Output(
627 name="cpPartialDefectsFromDarkCombined",
628 doc="Output measured defects.",
629 storageClass="Defects",
630 dimensions=("instrument", "detector"),
631 )
634class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig,
635 pipelineConnections=MeasureDefectsCombinedConnections):
636 """Configuration for measuring defects from combined exposures.
637 """
638 pass
641class MeasureDefectsCombinedTask(MeasureDefectsTask):
642 """Task to measure defects in combined images."""
644 ConfigClass = MeasureDefectsCombinedTaskConfig
645 _DefaultName = "cpDefectMeasureCombined"
648class MeasureDefectsCombinedWithFilterConnections(MeasureDefectsCombinedConnections,
649 dimensions=("instrument", "detector")):
650 """Task to measure defects in combined flats under a certain filter."""
651 inputExp = cT.Input(
652 name="flat",
653 doc="Input ISR-processed combined exposure to measure.",
654 storageClass="ExposureF",
655 dimensions=("instrument", "detector", "physical_filter"),
656 multiple=False,
657 isCalibration=True,
658 )
659 camera = cT.PrerequisiteInput(
660 name='camera',
661 doc="Camera associated with this exposure.",
662 storageClass="Camera",
663 dimensions=("instrument", ),
664 isCalibration=True,
665 )
667 outputDefects = cT.Output(
668 name="cpPartialDefectsFromFlatCombinedWithFilter",
669 doc="Output measured defects.",
670 storageClass="Defects",
671 dimensions=("instrument", "detector", "physical_filter"),
672 )
675class MeasureDefectsCombinedWithFilterTaskConfig(
676 MeasureDefectsTaskConfig,
677 pipelineConnections=MeasureDefectsCombinedWithFilterConnections):
678 """Configuration for measuring defects from combined exposures.
679 """
680 pass
683class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask):
684 """Task to measure defects in combined images."""
686 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig
687 _DefaultName = "cpDefectMeasureWithFilterCombined"
690class MergeDefectsConnections(pipeBase.PipelineTaskConnections,
691 dimensions=("instrument", "detector")):
692 inputDefects = cT.Input(
693 name="singleExpDefects",
694 doc="Measured defect lists.",
695 storageClass="Defects",
696 dimensions=("instrument", "detector", "exposure",),
697 multiple=True,
698 )
699 camera = cT.PrerequisiteInput(
700 name='camera',
701 doc="Camera associated with these defects.",
702 storageClass="Camera",
703 dimensions=("instrument", ),
704 isCalibration=True,
705 )
707 mergedDefects = cT.Output(
708 name="defects",
709 doc="Final merged defects.",
710 storageClass="Defects",
711 dimensions=("instrument", "detector"),
712 multiple=False,
713 isCalibration=True,
714 )
717class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig,
718 pipelineConnections=MergeDefectsConnections):
719 """Configuration for merging single exposure defects.
720 """
722 assertSameRun = pexConfig.Field(
723 dtype=bool,
724 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or "
725 "if the run key isn't found."),
726 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
727 )
728 ignoreFilters = pexConfig.Field(
729 dtype=bool,
730 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
731 " images. Allows mixing of filters in the input flats. Set to False if you think"
732 " your defects might be chromatic and want to have registry support for varying"
733 " defects with respect to filter."),
734 default=True,
735 )
736 nullFilterName = pexConfig.Field(
737 dtype=str,
738 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
739 default="NONE",
740 )
741 combinationMode = pexConfig.ChoiceField(
742 doc="Which types of defects to identify",
743 dtype=str,
744 default="FRACTION",
745 allowed={
746 "AND": "Logical AND the pixels found in each visit to form set ",
747 "OR": "Logical OR the pixels found in each visit to form set ",
748 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
749 }
750 )
751 combinationFraction = pexConfig.RangeField(
752 dtype=float,
753 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
754 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
755 " mode AND to require pixel to appear in all images."),
756 default=0.7,
757 min=0,
758 max=1,
759 )
760 edgesAsDefects = pexConfig.Field(
761 dtype=bool,
762 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
763 " Normal treatment is to simply exclude this region from the defect finding, such that no"
764 " defect will be located there."),
765 default=False,
766 )
769class MergeDefectsTask(pipeBase.PipelineTask):
770 """Merge the defects from multiple exposures.
771 """
773 ConfigClass = MergeDefectsTaskConfig
774 _DefaultName = 'cpDefectMerge'
776 def run(self, inputDefects, camera):
777 """Merge a list of single defects to find the common defect regions.
779 Parameters
780 ----------
781 inputDefects : `list` [`lsst.ip.isr.Defects`]
782 Partial defects from a single exposure.
783 camera : `lsst.afw.cameraGeom.Camera`
784 Camera to use for metadata.
786 Returns
787 -------
788 results : `lsst.pipe.base.Struct`
789 Results struct containing:
791 ``mergedDefects``
792 The defects merged from the input lists
793 (`lsst.ip.isr.Defects`).
794 """
795 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
796 if detectorId is None:
797 raise RuntimeError("Cannot identify detector id.")
798 detector = camera[detectorId]
800 imageTypes = set()
801 for inDefect in inputDefects:
802 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN')
803 imageTypes.add(imageType)
805 # Determine common defect pixels separately for each input image type.
806 splitDefects = list()
807 for imageType in imageTypes:
808 sumImage = afwImage.MaskedImageF(detector.getBBox())
809 count = 0
810 for inDefect in inputDefects:
811 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'):
812 count += 1
813 for defect in inDefect:
814 sumImage.image[defect.getBBox()] += 1.0
815 sumImage /= count
816 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
817 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType))
819 if self.config.combinationMode == 'AND':
820 threshold = 1.0
821 elif self.config.combinationMode == 'OR':
822 threshold = 0.0
823 elif self.config.combinationMode == 'FRACTION':
824 threshold = self.config.combinationFraction
825 else:
826 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}")
827 indices = np.where(sumImage.getImage().getArray() > threshold)
828 BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
829 sumImage.getMask().getArray()[indices] |= BADBIT
830 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType))
831 partialDefect = Defects.fromMask(sumImage, 'BAD')
832 splitDefects.append(partialDefect)
834 # Do final combination of separate image types
835 finalImage = afwImage.MaskedImageF(detector.getBBox())
836 for inDefect in splitDefects:
837 for defect in inDefect:
838 finalImage.image[defect.getBBox()] += 1
839 finalImage /= len(splitDefects)
840 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
841 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, ))
843 # This combination is the OR of all image types
844 threshold = 0.0
845 indices = np.where(finalImage.getImage().getArray() > threshold)
846 BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
847 finalImage.getMask().getArray()[indices] |= BADBIT
848 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), ))
850 if self.config.edgesAsDefects:
851 self.log.info("Masking edge pixels as defects.")
852 # Do the same as IsrTask.maskEdges()
853 box = detector.getBBox()
854 subImage = finalImage[box]
855 box.grow(-self.nPixBorder)
856 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
858 merged = Defects.fromMask(finalImage, 'BAD')
859 merged.updateMetadataFromExposures(inputDefects)
860 merged.updateMetadata(camera=camera, detector=detector, filterName=None,
861 setCalibId=True, setDate=True)
863 return pipeBase.Struct(
864 mergedDefects=merged,
865 )
867# Subclass the MergeDefects task to reduce the input dimensions
868# from ("instrument", "detector", "exposure") to
869# ("instrument", "detector").
872class MergeDefectsCombinedConnections(pipeBase.PipelineTaskConnections,
873 dimensions=("instrument", "detector")):
874 inputFlatDefects = cT.Input(
875 name="cpPartialDefectsFromDarkCombined",
876 doc="Measured defect lists.",
877 storageClass="Defects",
878 dimensions=("instrument", "detector",),
879 multiple=False,
880 )
881 inputDarkDefects = cT.Input(
882 name="cpPartialDefectsFromFlatCombinedWithFilter",
883 doc="Additional measured defect lists.",
884 storageClass="Defects",
885 dimensions=("instrument", "detector", "physical_filter"),
886 multiple=False,
887 )
888 camera = cT.PrerequisiteInput(
889 name='camera',
890 doc="Camera associated with these defects.",
891 storageClass="Camera",
892 dimensions=("instrument", ),
893 isCalibration=True,
894 )
896 mergedDefects = cT.Output(
897 name="defectsCombined",
898 doc="Final merged defects.",
899 storageClass="Defects",
900 dimensions=("instrument", "detector"),
901 multiple=False,
902 isCalibration=True,
903 )
906class MergeDefectsCombinedTaskConfig(MergeDefectsTaskConfig,
907 pipelineConnections=MergeDefectsCombinedConnections):
908 """Configuration for merging defects from combined exposure.
909 """
910 def validate(self):
911 super().validate()
912 if self.combinationMode != 'OR':
913 raise ValueError("combinationMode must be 'OR'")
916class MergeDefectsCombinedTask(MergeDefectsTask):
917 """Task to measure defects in combined images."""
919 ConfigClass = MergeDefectsCombinedTaskConfig
920 _DefaultName = "cpDefectMergeCombined"
922 def runQuantum(self, butlerQC, inputRefs, outputRefs):
923 inputs = butlerQC.get(inputRefs)
924 # Turn inputFlatDefects and inputDarkDefects into a list
925 # which is what MergeDefectsTask expects.
926 tempList = [inputs['inputFlatDefects'], inputs['inputDarkDefects']]
927 # Rename inputDefects
928 inputsCombined = {'inputDefects': tempList, 'camera': inputs['camera']}
930 outputs = super().run(**inputsCombined)
931 butlerQC.put(outputs, outputRefs)