Coverage for python/lsst/cp/pipe/defects.py: 20%
365 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-18 10:38 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-18 10:38 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask',
24 'MergeDefectsTaskConfig', 'MergeDefectsTask',
25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask',
26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ]
28import numpy as np
30import lsst.pipe.base as pipeBase
31import lsst.pipe.base.connectionTypes as cT
33from lsstDebug import getDebugFrame
34import lsst.pex.config as pexConfig
36import lsst.afw.image as afwImage
37import lsst.afw.math as afwMath
38import lsst.afw.detection as afwDetection
39import lsst.afw.display as afwDisplay
40from lsst.afw import cameraGeom
41from lsst.geom import Box2I, Point2I
42from lsst.meas.algorithms import SourceDetectionTask
43from lsst.ip.isr import Defects, countMaskedPixels
44from lsst.pex.exceptions import InvalidParameterError
47class MeasureDefectsConnections(pipeBase.PipelineTaskConnections,
48 dimensions=("instrument", "exposure", "detector")):
49 inputExp = cT.Input(
50 name="defectExps",
51 doc="Input ISR-processed exposures to measure.",
52 storageClass="Exposure",
53 dimensions=("instrument", "detector", "exposure"),
54 multiple=False
55 )
56 camera = cT.PrerequisiteInput(
57 name='camera',
58 doc="Camera associated with this exposure.",
59 storageClass="Camera",
60 dimensions=("instrument", ),
61 isCalibration=True,
62 )
64 outputDefects = cT.Output(
65 name="singleExpDefects",
66 doc="Output measured defects.",
67 storageClass="Defects",
68 dimensions=("instrument", "detector", "exposure"),
69 )
72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig,
73 pipelineConnections=MeasureDefectsConnections):
74 """Configuration for measuring defects from a list of exposures
75 """
77 thresholdType = pexConfig.ChoiceField(
78 dtype=str,
79 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found "
80 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found "
81 "in flats, and hot pixels in darks."),
82 default='STDEV',
83 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.",
84 'VALUE': "Use pixel value to determine detection threshold."},
85 )
86 darkCurrentThreshold = pexConfig.Field(
87 dtype=float,
88 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define "
89 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."),
90 default=5,
91 )
92 fracThresholdFlat = pexConfig.Field(
93 dtype=float,
94 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark "
95 "pixels in flat images (fraction of the mean value per amplifier)."
96 "Unused if thresholdType==``STDEV``."),
97 default=0.8,
98 )
99 nSigmaBright = pexConfig.Field(
100 dtype=float,
101 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot "
102 "pixel detection. The default value was found to be "
103 "appropriate for some LSST sensors in DM-17490. "
104 "Unused if thresholdType==``VALUE``"),
105 default=4.8,
106 )
107 nSigmaDark = pexConfig.Field(
108 dtype=float,
109 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel "
110 "detection. The default value was found to be "
111 "appropriate for some LSST sensors in DM-17490. "
112 "Unused if thresholdType==``VALUE``"),
113 default=-5.0,
114 )
115 nPixBorderUpDown = pexConfig.Field(
116 dtype=int,
117 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
118 default=7,
119 )
120 nPixBorderLeftRight = pexConfig.Field(
121 dtype=int,
122 doc="Number of pixels to exclude from left & right of image when looking for defects.",
123 default=7,
124 )
125 badOnAndOffPixelColumnThreshold = pexConfig.Field(
126 dtype=int,
127 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) "
128 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the "
129 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long "
130 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all "
131 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive "
132 "non-bad pixels in BPC, an exception to the above is made and those consecutive "
133 "'goodPixelColumnGapThreshold' are not marked as bad."),
134 default=50,
135 )
136 goodPixelColumnGapThreshold = pexConfig.Field(
137 dtype=int,
138 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see "
139 "'badOnAndOffPixelColumnThreshold')."),
140 default=30,
141 )
143 def validate(self):
144 super().validate()
145 if self.nSigmaBright < 0.0:
146 raise ValueError("nSigmaBright must be above 0.0.")
147 if self.nSigmaDark > 0.0:
148 raise ValueError("nSigmaDark must be below 0.0.")
151class MeasureDefectsTask(pipeBase.PipelineTask):
152 """Measure the defects from one exposure.
153 """
155 ConfigClass = MeasureDefectsTaskConfig
156 _DefaultName = 'cpDefectMeasure'
158 def run(self, inputExp, camera):
159 """Measure one exposure for defects.
161 Parameters
162 ----------
163 inputExp : `lsst.afw.image.Exposure`
164 Exposure to examine.
165 camera : `lsst.afw.cameraGeom.Camera`
166 Camera to use for metadata.
168 Returns
169 -------
170 results : `lsst.pipe.base.Struct`
171 Results struct containing:
173 ``outputDefects``
174 The defects measured from this exposure
175 (`lsst.ip.isr.Defects`).
176 """
177 detector = inputExp.getDetector()
178 try:
179 filterName = inputExp.getFilter().physicalLabel
180 except AttributeError:
181 filterName = None
183 defects = self._findHotAndColdPixels(inputExp)
185 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN')
186 msg = "Found %s defects containing %s pixels in %s"
187 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
189 defects.updateMetadataFromExposures([inputExp])
190 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName,
191 setCalibId=True, setDate=True,
192 cpDefectGenImageType=datasetType)
194 return pipeBase.Struct(
195 outputDefects=defects,
196 )
198 @staticmethod
199 def _nPixFromDefects(defects):
200 """Count pixels in a defect.
202 Parameters
203 ----------
204 defects : `lsst.ip.isr.Defects`
205 Defects to measure.
207 Returns
208 -------
209 nPix : `int`
210 Number of defect pixels.
211 """
212 nPix = 0
213 for defect in defects:
214 nPix += defect.getBBox().getArea()
215 return nPix
217 def _findHotAndColdPixels(self, exp):
218 """Find hot and cold pixels in an image.
220 Using config-defined thresholds on a per-amp basis, mask
221 pixels that are nSigma above threshold in dark frames (hot
222 pixels), or nSigma away from the clipped mean in flats (hot &
223 cold pixels).
225 Parameters
226 ----------
227 exp : `lsst.afw.image.exposure.Exposure`
228 The exposure in which to find defects.
230 Returns
231 -------
232 defects : `lsst.ip.isr.Defects`
233 The defects found in the image.
234 """
235 self._setEdgeBits(exp)
236 maskedIm = exp.maskedImage
238 # the detection polarity for afwDetection, True for positive,
239 # False for negative, and therefore True for darks as they only have
240 # bright pixels, and both for flats, as they have bright and dark pix
241 footprintList = []
243 hotPixelCount = {}
244 coldPixelCount = {}
246 for amp in exp.getDetector():
247 ampName = amp.getName()
249 hotPixelCount[ampName] = 0
250 coldPixelCount[ampName] = 0
252 ampImg = maskedIm[amp.getBBox()].clone()
254 # crop ampImage depending on where the amp lies in the image
255 if self.config.nPixBorderLeftRight:
256 if ampImg.getX0() == 0:
257 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
258 else:
259 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
260 if self.config.nPixBorderUpDown:
261 if ampImg.getY0() == 0:
262 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
263 else:
264 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
266 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
267 continue
269 # Remove a background estimate
270 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
271 ampImg -= meanClip
273 # Determine thresholds
274 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue()
275 expTime = exp.getInfo().getVisitInfo().getExposureTime()
276 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN')
277 if np.isnan(expTime):
278 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second",
279 expTime, ampName, datasetType)
280 expTime = 1.
281 thresholdType = self.config.thresholdType
282 if thresholdType == 'VALUE':
283 # LCA-128 and eoTest: bright/hot pixels in dark images are
284 # defined as any pixel with more than 5 e-/s of dark current.
285 # We scale by the exposure time.
286 if datasetType.lower() == 'dark':
287 # hot pixel threshold
288 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain()
289 else:
290 # LCA-128 and eoTest: dark/cold pixels in flat images as
291 # defined as any pixel with photoresponse <80% of
292 # the mean (at 500nm).
294 # We subtracted the mean above, so the threshold will be
295 # negative cold pixel threshold.
296 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip
297 # Find equivalent sigma values.
298 if stDev == 0.0:
299 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.",
300 stDev, ampName, datasetType)
301 nSigmaList = [np.inf]
302 else:
303 nSigmaList = [valueThreshold/stDev]
304 else:
305 hotPixelThreshold = self.config.nSigmaBright
306 coldPixelThreshold = self.config.nSigmaDark
307 if datasetType.lower() == 'dark':
308 nSigmaList = [hotPixelThreshold]
309 valueThreshold = stDev*hotPixelThreshold
310 else:
311 nSigmaList = [hotPixelThreshold, coldPixelThreshold]
312 valueThreshold = [x*stDev for x in nSigmaList]
314 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel"
315 "Values (hot and cold pixels thresholds): %s, %s",
316 datasetType, ampName, thresholdType, nSigmaList, valueThreshold)
317 mergedSet = None
318 for sigma in nSigmaList:
319 nSig = np.abs(sigma)
320 self.debugHistogram('ampFlux', ampImg, nSig, exp)
321 polarity = {-1: False, 1: True}[np.sign(sigma)]
323 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
325 try:
326 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
327 except InvalidParameterError:
328 # This occurs if the image sigma value is 0.0.
329 # Let's mask the whole area.
330 minValue = np.nanmin(ampImg.image.array) - 1.0
331 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True)
332 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
334 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
336 if mergedSet is None:
337 mergedSet = footprintSet
338 else:
339 mergedSet.merge(footprintSet)
341 if polarity:
342 # hot pixels
343 for fp in footprintSet.getFootprints():
344 hotPixelCount[ampName] += fp.getArea()
345 else:
346 # cold pixels
347 for fp in footprintSet.getFootprints():
348 coldPixelCount[ampName] += fp.getArea()
350 footprintList += mergedSet.getFootprints()
352 self.debugView('defectMap', ampImg,
353 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector())
355 defects = Defects.fromFootprintList(footprintList)
356 defects, count = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
357 defects.updateCounters(columns=count, hot=hotPixelCount, cold=coldPixelCount)
359 return defects
361 @staticmethod
362 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
363 """Return the number of non-bad pixels in the image."""
364 nPixels = maskedIm.mask.array.size
365 nBad = countMaskedPixels(maskedIm, badMaskString)
366 return nPixels - nBad
368 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
369 """Set edge bits on an exposure or maskedImage.
371 Raises
372 ------
373 TypeError
374 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
375 """
376 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
377 mi = exposureOrMaskedImage.maskedImage
378 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
379 mi = exposureOrMaskedImage
380 else:
381 t = type(exposureOrMaskedImage)
382 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
384 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
385 if self.config.nPixBorderLeftRight:
386 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
387 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
388 if self.config.nPixBorderUpDown:
389 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
390 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
392 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
393 """Mask blocks in a column if there are on-and-off bad pixels
395 If there's a column with on and off bad pixels, mask all the
396 pixels in between, except if there is a large enough gap of
397 consecutive good pixels between two bad pixels in the column.
399 Parameters
400 ----------
401 defects : `lsst.ip.isr.Defects`
402 The defects found in the image so far
404 Returns
405 -------
406 defects : `lsst.ip.isr.Defects`
407 If the number of bad pixels in a column is not larger or
408 equal than self.config.badPixelColumnThreshold, the input
409 list is returned. Otherwise, the defects list returned
410 will include boxes that mask blocks of on-and-of pixels.
411 badColumnCount : `int`
412 Number of bad columns masked.
413 """
414 badColumnCount = 0
415 # Get the (x, y) values of each bad pixel in amp.
416 coordinates = []
417 for defect in defects:
418 bbox = defect.getBBox()
419 x0, y0 = bbox.getMinX(), bbox.getMinY()
420 deltaX0, deltaY0 = bbox.getDimensions()
421 for j in np.arange(y0, y0+deltaY0):
422 for i in np.arange(x0, x0 + deltaX0):
423 coordinates.append((i, j))
425 x, y = [], []
426 for coordinatePair in coordinates:
427 x.append(coordinatePair[0])
428 y.append(coordinatePair[1])
430 x = np.array(x)
431 y = np.array(y)
432 # Find the defects with same "x" (vertical) coordinate (column).
433 unique, counts = np.unique(x, return_counts=True)
434 multipleX = []
435 for (a, b) in zip(unique, counts):
436 if b >= self.config.badOnAndOffPixelColumnThreshold:
437 multipleX.append(a)
438 if len(multipleX) != 0:
439 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
440 badColumnCount += 1
442 return defects, badColumnCount
444 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
445 """Mask blocks in a column if number of on-and-off bad pixels is above
446 threshold.
448 This function is called if the number of on-and-off bad pixels
449 in a column is larger or equal than
450 self.config.badOnAndOffPixelColumnThreshold.
452 Parameters
453 ---------
454 x : `list`
455 Lower left x coordinate of defect box. x coordinate is
456 along the short axis if amp.
457 y : `list`
458 Lower left y coordinate of defect box. x coordinate is
459 along the long axis if amp.
460 multipleX : list
461 List of x coordinates in amp. with multiple bad pixels
462 (i.e., columns with defects).
463 defects : `lsst.ip.isr.Defects`
464 The defcts found in the image so far
466 Returns
467 -------
468 defects : `lsst.ip.isr.Defects`
469 The defects list returned that will include boxes that
470 mask blocks of on-and-of pixels.
471 """
472 with defects.bulk_update():
473 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
474 for x0 in multipleX:
475 index = np.where(x == x0)
476 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
477 multipleY.sort() # Ensure that the y values are sorted to look for gaps.
478 minY, maxY = np.min(multipleY), np.max(multipleY)
479 # Next few lines: don't mask pixels in column if gap
480 # of good pixels between two consecutive bad pixels is
481 # larger or equal than 'goodPixelColumnGapThreshold'.
482 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
483 if len(diffIndex) != 0:
484 limits = [minY] # put the minimum first
485 for gapIndex in diffIndex:
486 limits.append(multipleY[gapIndex])
487 limits.append(multipleY[gapIndex+1])
488 limits.append(maxY) # maximum last
489 for i in np.arange(0, len(limits)-1, 2):
490 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
491 defects.append(s)
492 else: # No gap is large enough
493 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
494 defects.append(s)
495 return defects
497 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover
498 """Plot the defects found by the task.
500 Parameters
501 ----------
502 stepname : `str`
503 Debug frame to request.
504 ampImage : `lsst.afw.image.MaskedImage`
505 Amplifier image to display.
506 defects : `lsst.ip.isr.Defects`
507 The defects to plot.
508 detector : `lsst.afw.cameraGeom.Detector`
509 Detector holding camera geometry.
510 """
511 frame = getDebugFrame(self._display, stepname)
512 if frame:
513 disp = afwDisplay.Display(frame=frame)
514 disp.scale('asinh', 'zscale')
515 disp.setMaskTransparency(80)
516 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
518 maskedIm = ampImage.clone()
519 defects.maskPixels(maskedIm, "BAD")
521 mpDict = maskedIm.mask.getMaskPlaneDict()
522 for plane in mpDict.keys():
523 if plane in ['BAD']:
524 continue
525 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
527 disp.setImageColormap('gray')
528 disp.mtv(maskedIm)
529 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
530 prompt = "Press Enter to continue [c]... "
531 while True:
532 ans = input(prompt).lower()
533 if ans in ('', 'c', ):
534 break
536 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp):
537 """Make a histogram of the distribution of pixel values for
538 each amp.
540 The main image data histogram is plotted in blue. Edge
541 pixels, if masked, are in red. Note that masked edge pixels
542 do not contribute to the underflow and overflow numbers.
544 Note that this currently only supports the 16-amp LSST
545 detectors.
547 Parameters
548 ----------
549 stepname : `str`
550 Debug frame to request.
551 ampImage : `lsst.afw.image.MaskedImage`
552 Amplifier image to display.
553 nSigmaUsed : `float`
554 The number of sigma used for detection
555 exp : `lsst.afw.image.exposure.Exposure`
556 The exposure in which the defects were found.
557 """
558 frame = getDebugFrame(self._display, stepname)
559 if frame:
560 import matplotlib.pyplot as plt
562 detector = exp.getDetector()
563 nX = np.floor(np.sqrt(len(detector)))
564 nY = len(detector) // nX
565 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10))
567 expTime = exp.getInfo().getVisitInfo().getExposureTime()
569 for (amp, a) in zip(reversed(detector), ax.flatten()):
570 mi = exp.maskedImage[amp.getBBox()]
572 # normalize by expTime as we plot in ADU/s and don't
573 # always work with master calibs
574 mi.image.array /= expTime
575 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
576 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
577 # Get array of pixels
578 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
579 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
580 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
582 thrUpper = mean + nSigmaUsed*sigma
583 thrLower = mean - nSigmaUsed*sigma
585 nRight = len(imgData[imgData > thrUpper])
586 nLeft = len(imgData[imgData < thrLower])
588 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
589 leftEdge = mean - nsig * nSigmaUsed*sigma
590 rightEdge = mean + nsig * nSigmaUsed*sigma
591 nbins = np.linspace(leftEdge, rightEdge, 1000)
592 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins,
593 lw=1, edgecolor='red')
594 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins,
595 lw=3, edgecolor='blue')
597 # Report number of entries in over- and under-flow
598 # bins, i.e. off the edges of the histogram
599 nOverflow = len(imgData[imgData > rightEdge])
600 nUnderflow = len(imgData[imgData < leftEdge])
602 # Put v-lines and textboxes in
603 a.axvline(thrUpper, c='k')
604 a.axvline(thrLower, c='k')
605 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
606 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
607 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
608 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
610 # set axis limits and scales
611 a.set_ylim([1., 1.7*np.max(y)])
612 lPlot, rPlot = a.get_xlim()
613 a.set_xlim(np.array([lPlot, rPlot]))
614 a.set_yscale('log')
615 a.set_xlabel("ADU/s")
616 fig.show()
617 prompt = "Press Enter or c to continue [chp]..."
618 while True:
619 ans = input(prompt).lower()
620 if ans in ("", " ", "c",):
621 break
622 elif ans in ("p", ):
623 import pdb
624 pdb.set_trace()
625 elif ans in ("h", ):
626 print("[h]elp [c]ontinue [p]db")
627 plt.close()
630class MeasureDefectsCombinedConnections(pipeBase.PipelineTaskConnections,
631 dimensions=("instrument", "detector")):
632 inputExp = cT.Input(
633 name="dark",
634 doc="Input ISR-processed combined exposure to measure.",
635 storageClass="ExposureF",
636 dimensions=("instrument", "detector"),
637 multiple=False,
638 isCalibration=True,
639 )
640 camera = cT.PrerequisiteInput(
641 name='camera',
642 doc="Camera associated with this exposure.",
643 storageClass="Camera",
644 dimensions=("instrument", ),
645 isCalibration=True,
646 )
648 outputDefects = cT.Output(
649 name="cpPartialDefectsFromDarkCombined",
650 doc="Output measured defects.",
651 storageClass="Defects",
652 dimensions=("instrument", "detector"),
653 )
656class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig,
657 pipelineConnections=MeasureDefectsCombinedConnections):
658 """Configuration for measuring defects from combined exposures.
659 """
660 pass
663class MeasureDefectsCombinedTask(MeasureDefectsTask):
664 """Task to measure defects in combined images."""
666 ConfigClass = MeasureDefectsCombinedTaskConfig
667 _DefaultName = "cpDefectMeasureCombined"
670class MeasureDefectsCombinedWithFilterConnections(pipeBase.PipelineTaskConnections,
671 dimensions=("instrument", "detector", "physical_filter")):
672 """Task to measure defects in combined flats under a certain filter."""
673 inputExp = cT.Input(
674 name="flat",
675 doc="Input ISR-processed combined exposure to measure.",
676 storageClass="ExposureF",
677 dimensions=("instrument", "detector", "physical_filter"),
678 multiple=False,
679 isCalibration=True,
680 )
681 camera = cT.PrerequisiteInput(
682 name='camera',
683 doc="Camera associated with this exposure.",
684 storageClass="Camera",
685 dimensions=("instrument", ),
686 isCalibration=True,
687 )
689 outputDefects = cT.Output(
690 name="cpPartialDefectsFromFlatCombinedWithFilter",
691 doc="Output measured defects.",
692 storageClass="Defects",
693 dimensions=("instrument", "detector", "physical_filter"),
694 )
697class MeasureDefectsCombinedWithFilterTaskConfig(
698 MeasureDefectsTaskConfig,
699 pipelineConnections=MeasureDefectsCombinedWithFilterConnections):
700 """Configuration for measuring defects from combined exposures.
701 """
702 pass
705class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask):
706 """Task to measure defects in combined images."""
708 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig
709 _DefaultName = "cpDefectMeasureWithFilterCombined"
712class MergeDefectsConnections(pipeBase.PipelineTaskConnections,
713 dimensions=("instrument", "detector")):
714 inputDefects = cT.Input(
715 name="singleExpDefects",
716 doc="Measured defect lists.",
717 storageClass="Defects",
718 dimensions=("instrument", "detector", "exposure",),
719 multiple=True,
720 )
721 camera = cT.PrerequisiteInput(
722 name='camera',
723 doc="Camera associated with these defects.",
724 storageClass="Camera",
725 dimensions=("instrument", ),
726 isCalibration=True,
727 )
729 mergedDefects = cT.Output(
730 name="defects",
731 doc="Final merged defects.",
732 storageClass="Defects",
733 dimensions=("instrument", "detector"),
734 multiple=False,
735 isCalibration=True,
736 )
739class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig,
740 pipelineConnections=MergeDefectsConnections):
741 """Configuration for merging single exposure defects.
742 """
744 assertSameRun = pexConfig.Field(
745 dtype=bool,
746 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or "
747 "if the run key isn't found."),
748 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
749 )
750 ignoreFilters = pexConfig.Field(
751 dtype=bool,
752 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
753 " images. Allows mixing of filters in the input flats. Set to False if you think"
754 " your defects might be chromatic and want to have registry support for varying"
755 " defects with respect to filter."),
756 default=True,
757 )
758 nullFilterName = pexConfig.Field(
759 dtype=str,
760 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
761 default="NONE",
762 )
763 combinationMode = pexConfig.ChoiceField(
764 doc="Which types of defects to identify",
765 dtype=str,
766 default="FRACTION",
767 allowed={
768 "AND": "Logical AND the pixels found in each visit to form set ",
769 "OR": "Logical OR the pixels found in each visit to form set ",
770 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
771 }
772 )
773 combinationFraction = pexConfig.RangeField(
774 dtype=float,
775 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
776 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
777 " mode AND to require pixel to appear in all images."),
778 default=0.7,
779 min=0,
780 max=1,
781 )
782 edgesAsDefects = pexConfig.Field(
783 dtype=bool,
784 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
785 " Normal treatment is to simply exclude this region from the defect finding, such that no"
786 " defect will be located there."),
787 default=False,
788 )
791class MergeDefectsTask(pipeBase.PipelineTask):
792 """Merge the defects from multiple exposures.
793 """
795 ConfigClass = MergeDefectsTaskConfig
796 _DefaultName = 'cpDefectMerge'
798 def run(self, inputDefects, camera):
799 """Merge a list of single defects to find the common defect regions.
801 Parameters
802 ----------
803 inputDefects : `list` [`lsst.ip.isr.Defects`]
804 Partial defects from a single exposure.
805 camera : `lsst.afw.cameraGeom.Camera`
806 Camera to use for metadata.
808 Returns
809 -------
810 results : `lsst.pipe.base.Struct`
811 Results struct containing:
813 ``mergedDefects``
814 The defects merged from the input lists
815 (`lsst.ip.isr.Defects`).
816 """
817 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None)
818 if detectorId is None:
819 raise RuntimeError("Cannot identify detector id.")
820 detector = camera[detectorId]
822 imageTypes = set()
823 for inDefect in inputDefects:
824 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN')
825 imageTypes.add(imageType)
827 # Determine common defect pixels separately for each input image type.
828 splitDefects = list()
829 for imageType in imageTypes:
830 sumImage = afwImage.MaskedImageF(detector.getBBox())
831 count = 0
832 for inDefect in inputDefects:
833 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'):
834 count += 1
835 for defect in inDefect:
836 sumImage.image[defect.getBBox()] += 1.0
837 sumImage /= count
838 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0])
839 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType))
841 if self.config.combinationMode == 'AND':
842 threshold = 1.0
843 elif self.config.combinationMode == 'OR':
844 threshold = 0.0
845 elif self.config.combinationMode == 'FRACTION':
846 threshold = self.config.combinationFraction
847 else:
848 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}")
849 indices = np.where(sumImage.getImage().getArray() > threshold)
850 BADBIT = sumImage.getMask().getPlaneBitMask('BAD')
851 sumImage.getMask().getArray()[indices] |= BADBIT
852 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType))
853 partialDefect = Defects.fromMask(sumImage, 'BAD')
854 splitDefects.append(partialDefect)
856 # Do final combination of separate image types
857 finalImage = afwImage.MaskedImageF(detector.getBBox())
858 for inDefect in splitDefects:
859 for defect in inDefect:
860 finalImage.image[defect.getBBox()] += 1
861 finalImage /= len(splitDefects)
862 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0])
863 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, ))
865 # This combination is the OR of all image types
866 threshold = 0.0
867 indices = np.where(finalImage.getImage().getArray() > threshold)
868 BADBIT = finalImage.getMask().getPlaneBitMask('BAD')
869 finalImage.getMask().getArray()[indices] |= BADBIT
870 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), ))
872 if self.config.edgesAsDefects:
873 self.log.info("Masking edge pixels as defects.")
874 # Do the same as IsrTask.maskEdges()
875 box = detector.getBBox()
876 subImage = finalImage[box]
877 box.grow(-self.nPixBorder)
878 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT)
880 merged = Defects.fromMask(finalImage, 'BAD')
881 merged.updateMetadataFromExposures(inputDefects)
882 merged.updateMetadata(camera=camera, detector=detector, filterName=None,
883 setCalibId=True, setDate=True)
885 return pipeBase.Struct(
886 mergedDefects=merged,
887 )
889# Subclass the MergeDefects task to reduce the input dimensions
890# from ("instrument", "detector", "exposure") to
891# ("instrument", "detector").
894class MergeDefectsCombinedConnections(pipeBase.PipelineTaskConnections,
895 dimensions=("instrument", "detector")):
896 inputDarkDefects = cT.Input(
897 name="cpPartialDefectsFromDarkCombined",
898 doc="Measured defect lists.",
899 storageClass="Defects",
900 dimensions=("instrument", "detector",),
901 multiple=True,
902 )
903 inputFlatDefects = cT.Input(
904 name="cpPartialDefectsFromFlatCombinedWithFilter",
905 doc="Additional measured defect lists.",
906 storageClass="Defects",
907 dimensions=("instrument", "detector", "physical_filter"),
908 multiple=True,
909 )
910 camera = cT.PrerequisiteInput(
911 name='camera',
912 doc="Camera associated with these defects.",
913 storageClass="Camera",
914 dimensions=("instrument", ),
915 isCalibration=True,
916 )
918 mergedDefects = cT.Output(
919 name="defects",
920 doc="Final merged defects.",
921 storageClass="Defects",
922 dimensions=("instrument", "detector"),
923 multiple=False,
924 isCalibration=True,
925 )
928class MergeDefectsCombinedTaskConfig(MergeDefectsTaskConfig,
929 pipelineConnections=MergeDefectsCombinedConnections):
930 """Configuration for merging defects from combined exposure.
931 """
932 def validate(self):
933 super().validate()
934 if self.combinationMode != 'OR':
935 raise ValueError("combinationMode must be 'OR'")
938class MergeDefectsCombinedTask(MergeDefectsTask):
939 """Task to measure defects in combined images."""
941 ConfigClass = MergeDefectsCombinedTaskConfig
942 _DefaultName = "cpDefectMergeCombined"
944 @staticmethod
945 def chooseBest(inputs):
946 """Select the input with the most exposures used."""
947 best = 0
948 if len(inputs) > 1:
949 nInput = 0
950 for num, exp in enumerate(inputs):
951 # This technically overcounts by a factor of 3.
952 N = len([k for k, v in exp.getMetadata().toDict().items() if "CPP_INPUT_" in k])
953 if N > nInput:
954 best = num
955 nInput = N
956 return inputs[best]
958 def runQuantum(self, butlerQC, inputRefs, outputRefs):
959 inputs = butlerQC.get(inputRefs)
960 # Turn inputFlatDefects and inputDarkDefects into a list which
961 # is what MergeDefectsTask expects. If there are multiple,
962 # use the one with the most inputs.
963 tempList = [self.chooseBest(inputs['inputFlatDefects']),
964 self.chooseBest(inputs['inputDarkDefects'])]
966 # Rename inputDefects
967 inputsCombined = {'inputDefects': tempList, 'camera': inputs['camera']}
969 outputs = super().run(**inputsCombined)
970 butlerQC.put(outputs, outputRefs)