23 __all__ = [
'FindDefectsTask',
24 'FindDefectsTaskConfig', ]
30 import lsst.pex.config
as pexConfig
40 from .utils
import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \
45 """Config class for defect finding""" 47 isrForFlats = pexConfig.ConfigurableField(
49 doc=
"Task to perform instrumental signature removal",
51 isrForDarks = pexConfig.ConfigurableField(
53 doc=
"Task to perform instrumental signature removal",
55 isrMandatoryStepsFlats = pexConfig.ListField(
57 doc=(
"isr operations that must be performed for valid results when using flats." 58 " Raises if any of these are False"),
59 default=[
'doAssembleCcd',
'doFringe']
61 isrMandatoryStepsDarks = pexConfig.ListField(
63 doc=(
"isr operations that must be performed for valid results when using darks. " 64 "Raises if any of these are False"),
65 default=[
'doAssembleCcd',
'doFringe']
67 isrForbiddenStepsFlats = pexConfig.ListField(
69 doc=(
"isr operations that must NOT be performed for valid results when using flats." 70 " Raises if any of these are True"),
71 default=[
'doAddDistortionModel',
'doBrighterFatter',
'doUseOpticsTransmission',
72 'doUseFilterTransmission',
'doUseSensorTransmission',
'doUseAtmosphereTransmission']
74 isrForbiddenStepsDarks = pexConfig.ListField(
76 doc=(
"isr operations that must NOT be performed for valid results when using darks." 77 " Raises if any of these are True"),
78 default=[
'doAddDistortionModel',
'doBrighterFatter',
'doUseOpticsTransmission',
79 'doUseFilterTransmission',
'doUseSensorTransmission',
'doUseAtmosphereTransmission']
81 isrDesirableSteps = pexConfig.ListField(
83 doc=(
"isr operations that it is advisable to perform, but are not mission-critical." 84 " WARNs are logged for any of these found to be False."),
87 ccdKey = pexConfig.Field(
89 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
92 imageTypeKey = pexConfig.Field(
94 doc=
"The key for the butler to use by which to check whether images are darks or flats",
97 mode = pexConfig.ChoiceField(
98 doc=(
"Use single master calibs (flat and dark) for finding defects, or a list of raw visits?" 99 " If MASTER, a single visit number should be supplied, for which the corresponding master flat" 100 " and dark will be used. If VISITS, the list of visits will be used, treating the flats and " 101 " darks as appropriate, depending on their image types, as determined by their imageType from" 102 " config.imageTypeKey"),
106 "VISITS":
"Calculate defects from a list of raw visits",
107 "MASTER":
"Use the corresponding master calibs from the specified visit to measure defects",
110 nSigmaBright = pexConfig.Field(
112 doc=(
"Number of sigma above mean for bright pixel detection. The default value was found to be",
113 " appropriate for some LSST sensors in DM-17490."),
116 nSigmaDark = pexConfig.Field(
118 doc=(
"Number of sigma below mean for dark pixel detection. The default value was found to be",
119 " appropriate for some LSST sensors in DM-17490."),
122 nPixBorderUpDown = pexConfig.Field(
124 doc=
"Number of pixels to exclude from top & bottom of image when looking for defects.",
127 nPixBorderLeftRight = pexConfig.Field(
129 doc=
"Number of pixels to exclude from left & right of image when looking for defects.",
132 edgesAsDefects = pexConfig.Field(
134 doc=(
"Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 135 " Normal treatment is to simply exclude this region from the defect finding, such that no" 136 " defect will be located there."),
139 assertSameRun = pexConfig.Field(
141 doc=(
"Ensure that all visits are from the same run? Raises if this is not the case, or" 142 "if the run key isn't found."),
145 combinationMode = pexConfig.ChoiceField(
146 doc=
"Which types of defects to identify",
150 "AND":
"Logical AND the pixels found in each visit to form set",
151 "OR":
"Logical OR the pixels found in each visit to form set",
152 "FRACTION":
"Use pixels found in more than config.combinationFraction of visits",
155 combinationFraction = pexConfig.RangeField(
157 doc=(
"The fraction (0..1) of visits in which a pixel was found to be defective across" 158 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 159 " mode AND to require pixel to appear in all images."),
164 makePlots = pexConfig.Field(
166 doc=(
"Plot histograms for each visit for each amp (one plot per detector) and the final" 167 " defects overlaid on the sensor."),
170 writeAs = pexConfig.ChoiceField(
171 doc=
"Write the output file as ASCII or FITS table",
175 "ASCII":
"Write the output as an ASCII file",
176 "FITS":
"Write the output as an FITS table",
177 "BOTH":
"Write the output as both a FITS table and an ASCII file",
183 """Task for finding defects in sensors. 185 The task has two modes of operation, defect finding in raws and in 186 master calibrations, which work as follows. 188 Master calib defect finding 189 ---------------------------- 191 A single visit number is supplied, for which the corresponding flat & dark 192 will be used. This is because, at present at least, there is no way to pass 193 a calibration exposure ID from the command line to a command line task. 195 The task retrieves the corresponding dark and flat exposures for the 196 supplied visit. If a flat is available the task will (be able to) look 197 for both bright and dark defects. If only a dark is found then only bright 198 defects will be sought. 200 All pixels above/below the specified nSigma which lie with the specified 201 borders for flats/darks are identified as defects. 203 Raw visit defect finding 204 ------------------------ 206 A list of exposure IDs are supplied for defect finding. The task will 207 detect bright pixels in the dark frames, if supplied, and bright & dark 208 pixels in the flats, if supplied, i.e. if you only supply darks you will 209 only be given bright defects. This is done automatically from the imageType 210 of the exposure, so the input exposure list can be a mix. 212 As with the master calib detection, all pixels above/below the specified 213 nSigma which lie with the specified borders for flats/darks are identified 214 as defects. Then, a post-processing step is done to merge these detections, 215 with pixels appearing in a fraction [0..1] of the images are kept as defects 216 and those appearing below that occurrence-threshold are discarded. 219 RunnerClass = SingleVisitListTaskRunner
220 ConfigClass = FindDefectsTaskConfig
221 _DefaultName =
"findDefects" 224 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
225 self.makeSubtask(
"isrForFlats")
226 self.makeSubtask(
"isrForDarks")
229 self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps)
231 self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps)
232 self.config.validate()
236 def _makeArgumentParser(cls):
237 """Augment argument parser for the FindDefectsTask.""" 239 parser.add_argument(
"--visitList", dest=
"visitList", nargs=
"*",
240 help=(
"List of visits to use. Same for each detector." 241 " Uses the normal 0..10:3^234 syntax"))
242 parser.add_id_argument(
"--id", datasetType=
"newDefects",
243 ContainerClass=NonexistentDatasetTaskDataIdContainer,
244 help=
"The ccds to use, e.g. --id ccd=0..100")
249 """Run the defect finding task. 251 Find the defects, as described in the main task docstring, from a 252 dataRef and a list of visit(s). 256 dataRef : `lsst.daf.persistence.ButlerDataRef` 257 dataRef for the detector for the visits to be fit. 258 visitList : `list` [`int`] 259 List of visits to be processed. If config.mode == 'VISITS' then the 260 list of visits is used. If config.mode == 'MASTER' then the length 261 of visitList must be one, and the corresponding master calibrations 266 result : `lsst.pipe.base.Struct` 267 Result struct with Components: 269 - ``defects`` : `lsst.meas.algorithms.Defect` 270 The defects found by the task. 271 - ``exitStatus`` : `int` 275 detNum = dataRef.dataId[self.config.ccdKey]
276 self.log.info(
"Calculating defects using %s visits for detector %s" % (visitList, detNum))
278 defectLists = {
'dark': [],
'flat': []}
280 if self.config.mode ==
'MASTER':
281 if len(visitList) > 1:
282 raise RuntimeError(f
"Must only specify one visit when using mode MASTER, got {visitList}")
283 dataRef.dataId[
'visit'] = visitList[0]
285 for datasetType
in defectLists.keys():
286 exp = dataRef.get(datasetType)
289 msg =
"Found %s defects containing %s pixels in master %s" 290 self.log.info(msg, len(defects), self.
_nPixFromDefects(defects), datasetType)
291 defectLists[datasetType].append(defects)
292 if self.config.makePlots:
294 defects, datasetType)
296 elif self.config.mode ==
'VISITS':
297 butler = dataRef.getButler()
299 if self.config.assertSameRun:
302 raise RuntimeError(f
"Got data from runs {runs} with assertSameRun==True")
304 for visit
in visitList:
305 imageType = butler.queryMetadata(
'raw', self.config.imageTypeKey, dataId={
'visit': visit})[0]
306 imageType = imageType.lower()
307 dataRef.dataId[
'visit'] = visit
309 if imageType ==
'flat':
310 exp = self.isrForFlats.
runDataRef(dataRef).exposure
312 defectLists[
'flat'].append(defects)
314 elif imageType ==
'dark':
315 exp = self.isrForDarks.
runDataRef(dataRef).exposure
317 defectLists[
'dark'].append(defects)
320 raise RuntimeError(f
"Failed on imageType {imageType}. Only flats and darks supported")
322 msg =
"Found %s defects containing %s pixels in visit %s" 325 if self.config.makePlots:
328 msg =
"Combining %s defect sets from darks for detector %s" 329 self.log.info(msg, len(defectLists[
'dark']), detNum)
331 self.config.combinationMode)
332 msg =
"Combining %s defect sets from flats for detector %s" 333 self.log.info(msg, len(defectLists[
'flat']), detNum)
335 self.config.combinationMode)
337 msg =
"Combining bright and dark defect sets for detector %s" 338 self.log.info(msg, detNum)
339 brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats]
344 self.log.info(
"Finished finding defects in detector %s" % detNum)
345 return pipeBase.Struct(defects=allDefects, exitStatus=0)
347 def _getNsigmaForPlot(self, imageType):
348 assert imageType
in [
'flat',
'dark']
349 nSig = self.config.nSigmaBright
if imageType ==
'flat' else self.config.nSigmaDark
353 def _nPixFromDefects(defect):
354 """Count the number of pixels in a defect object.""" 357 nPix += d.getBBox().getArea()
360 def _writeData(self, dataRef, defects):
361 """Write the data out to the defect file. 365 dataRef : `lsst.daf.persistence.ButlerDataRef` 366 dataRef for the detector for defects to be written. 367 defects : `lsst.meas.algorithms.Defect` 368 The defects to be written. 370 filename = dataRef.getUri(write=
True)
371 dirname = os.path.dirname(filename)
372 if not os.path.exists(dirname):
375 msg =
"Writing defects to %s in format: %s" 376 self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs)
378 if self.config.writeAs
in [
'FITS',
'BOTH']:
379 defects.writeFits(filename)
380 if self.config.writeAs
in [
'ASCII',
'BOTH']:
381 wroteTo = defects.writeText(filename)
382 assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0])
386 def _getRunListFromVisits(butler, visitList):
387 """Return the set of runs for the visits in visitList.""" 389 for visit
in visitList:
390 runs.add(butler.queryMetadata(
'raw',
'run', dataId={
'visit': visit})[0])
393 def _postProcessDefectSets(self, defectList, imageDimensions, mode):
394 """Combine a list of defects to make a single defect object. 396 AND, OR or use percentage of visits in which defects appear 401 defectList : `list` [`lsst.meas.algorithms.Defect`] 402 The lList of defects to merge. 403 imageDimensions : `tuple` [`int`] 404 The size of the image. 406 The combination mode to use, either 'AND', 'OR' or 'FRACTION' 410 defects : `lsst.meas.algorithms.Defect` 411 The defect set resulting from the merge. 418 if len(defectList) == 1:
421 sumImage = afwImage.MaskedImageF(imageDimensions)
422 for defects
in defectList:
423 for defect
in defects:
424 sumImage.image[defect.getBBox()] += 1
425 sumImage /= len(defectList)
427 nDetected = len(np.where(sumImage.image.array > 0)[0])
428 self.log.info(
"Pre-merge %s pixels with non-zero detections" % nDetected)
431 indices = np.where(sumImage.image.array > 0)
435 elif mode ==
'FRACTION':
436 threshold = self.config.combinationFraction
438 raise RuntimeError(f
"Got unsupported combinationMode {mode}")
439 indices = np.where(sumImage.image.array >= threshold)
441 BADBIT = sumImage.mask.getPlaneBitMask(
'BAD')
442 sumImage.mask.array[indices] |= BADBIT
444 self.log.info(
"Post-merge %s pixels marked as defects" % len(indices[0]))
446 if self.config.edgesAsDefects:
447 self.log.info(
"Masking edge pixels as defects in addition to previously identified defects")
450 defects = measAlg.Defects.fromMask(sumImage,
'BAD')
454 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
455 """Return the number of non-bad pixels in the image.""" 456 nPixels = maskedIm.mask.array.size
458 return nPixels - nBad
461 """Find hot and cold pixels in an image. 463 Using config-defined thresholds on a per-amp basis, mask pixels 464 that are nSigma above threshold in dark frames (hot pixels), 465 or nSigma away from the clipped mean in flats (hot & cold pixels). 469 exp : `lsst.afw.image.exposure.Exposure` 470 The exposure in which to find defects. 472 The image type, either 'dark' or 'flat'. 474 If true, update exp with hot and cold pixels. 476 cold: DETECTED_NEGATIVE 480 defects : `lsst.meas.algorithms.Defect` 481 The defects found in the image. 483 assert imageType
in [
'flat',
'dark']
486 maskedIm = exp.maskedImage
491 polarities = {
'dark': [
True],
'flat': [
True,
False]}[imageType]
495 for amp
in exp.getDetector():
496 ampImg = maskedIm[amp.getBBox()].clone()
499 if self.config.nPixBorderLeftRight:
500 if ampImg.getX0() == 0:
501 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
503 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
504 if self.config.nPixBorderUpDown:
505 if ampImg.getY0() == 0:
506 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
508 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
513 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
516 for polarity
in polarities:
517 nSig = self.config.nSigmaBright
if polarity
else self.config.nSigmaDark
518 threshold = afwDetection.createThreshold(nSig,
'stdev', polarity=polarity)
520 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
522 footprintSet.setMask(maskedIm.mask, (
"DETECTED" if polarity
else "DETECTED_NEGATIVE"))
524 if mergedSet
is None:
525 mergedSet = footprintSet
527 mergedSet.merge(footprintSet)
529 footprintList += mergedSet.getFootprints()
531 defects = measAlg.Defects.fromFootprintList(footprintList)
534 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
535 """Set edge bits on an exposure or maskedImage. 540 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 542 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
543 mi = exposureOrMaskedImage.maskedImage
544 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
545 mi = exposureOrMaskedImage
547 t = type(exposureOrMaskedImage)
548 raise TypeError(f
"Function supports exposure or maskedImage but not {t}")
550 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
551 if self.config.nPixBorderLeftRight:
552 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
553 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
554 if self.config.nPixBorderUpDown:
555 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
556 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
558 def _plot(self, dataRef, exp, visit, nSig, defects, imageType):
559 """Plot the defects and pixel histograms. 563 dataRef : `lsst.daf.persistence.ButlerDataRef` 564 dataRef for the detector. 565 exp : `lsst.afw.image.exposure.Exposure` 566 The exposure in which the defects were found. 570 The number of sigma used for detection 571 defects : `lsst.meas.algorithms.Defect` 574 The type of image, either 'dark' or 'flat'. 576 Currently only for LSST sensors. Plots are written to the path 577 given by the butler for the ``cpPipePlotRoot`` dataset type. 579 import matplotlib.pyplot
as plt
580 from matplotlib.backends.backend_pdf
import PdfPages
582 afwDisplay.setDefaultBackend(
"matplotlib")
583 plt.interactive(
False)
585 dirname = dataRef.getUri(datasetType=
'cpPipePlotRoot', write=
True)
586 if not os.path.exists(dirname):
589 detNum = exp.getDetector().getId()
590 nAmps = len(exp.getDetector())
592 if self.config.mode ==
"MASTER":
593 filename = f
"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf" 594 elif self.config.mode ==
"VISITS":
595 filename = f
"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf" 597 filenameFull = os.path.join(dirname, filename)
599 with warnings.catch_warnings():
600 msg =
"Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure." 601 warnings.filterwarnings(
"ignore", message=msg)
602 with PdfPages(filenameFull)
as pdfPages:
609 self.log.info(
"Wrote plot(s) to %s" % filenameFull)
611 def _plotDefects(self, exp, visit, defects, imageType):
612 """Plot the defects found by the task. 616 exp : `lsst.afw.image.exposure.Exposure` 617 The exposure in which the defects were found. 620 defects : `lsst.meas.algorithms.Defect` 623 The type of image, either 'dark' or 'flat'. 625 expCopy = exp.clone()
627 maskedIm = expCopy.maskedImage
629 defects.maskPixels(expCopy.maskedImage,
"BAD")
630 detector = expCopy.getDetector()
632 disp = afwDisplay.Display(0, reopenPlot=
True, dpi=200)
634 if imageType ==
"flat":
636 ampIm = maskedIm.image[amp.getBBox()]
637 ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1
639 mpDict = maskedIm.mask.getMaskPlaneDict()
640 for plane
in mpDict.keys():
643 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
645 disp.scale(
'asinh',
'zscale')
646 disp.setMaskTransparency(80)
647 disp.setMaskPlaneColor(
"BAD", afwDisplay.RED)
649 disp.setImageColormap(
'gray')
650 title = (f
"Detector: {detector.getName()[-3:]} {detector.getSerial()}" 651 f
", Type: {imageType}, visit: {visit}")
652 disp.mtv(maskedIm, title=title)
654 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=
True, display=disp)
656 def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed):
658 Make a histogram of the distribution of pixel values for each amp. 660 The main image data histogram is plotted in blue. Edge pixels, 661 if masked, are in red. Note that masked edge pixels do not contribute 662 to the underflow and overflow numbers. 664 Note that this currently only supports the 16-amp LSST detectors. 668 dataRef : `lsst.daf.persistence.ButlerDataRef` 669 dataRef for the detector. 670 exp : `lsst.afw.image.exposure.Exposure` 671 The exposure in which the defects were found. 675 The number of sigma used for detection 677 import matplotlib.pyplot
as plt
679 detector = exp.getDetector()
681 if len(detector) != 16:
682 raise RuntimeError(
"Plotting currently only supported for 16 amp detectors")
683 fig, ax = plt.subplots(nrows=4, ncols=4, sharex=
'col', sharey=
'row', figsize=(13, 10))
685 expTime = exp.getInfo().getVisitInfo().getExposureTime()
687 for (amp, a)
in zip(reversed(detector), ax.flatten()):
688 mi = exp.maskedImage[amp.getBBox()]
691 mi.image.array /= expTime
692 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
693 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
696 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask(
"EDGE")
697 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
698 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
700 thrUpper = mean + nSigmaUsed*sigma
701 thrLower = mean - nSigmaUsed*sigma
703 nRight = len(imgData[imgData > thrUpper])
704 nLeft = len(imgData[imgData < thrLower])
706 nsig = nSigmaUsed + 1.2
707 leftEdge = mean - nsig * nSigmaUsed*sigma
708 rightEdge = mean + nsig * nSigmaUsed*sigma
709 nbins = np.linspace(leftEdge, rightEdge, 1000)
710 ey, bin_borders, patches = a.hist(edgeData, histtype=
'step', bins=nbins, lw=1, edgecolor=
'red')
711 y, bin_borders, patches = a.hist(imgData, histtype=
'step', bins=nbins, lw=3, edgecolor=
'blue')
714 nOverflow = len(imgData[imgData > rightEdge])
715 nUnderflow = len(imgData[imgData < leftEdge])
718 a.axvline(thrUpper, c=
'k')
719 a.axvline(thrLower, c=
'k')
720 msg = f
"{amp.getName()}\nmean:{mean: .1f}\n$\\sigma$:{sigma: .1f}" 721 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
722 msg = f
"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 723 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
726 a.set_ylim([1., 1.7*np.max(y)])
727 lPlot, rPlot = a.get_xlim()
728 a.set_xlim(np.array([lPlot, rPlot]))
730 a.set_xlabel(
"ADU/s")
def _getNsigmaForPlot(self, imageType)
def runDataRef(self, dataRef, visitList)
def _getRunListFromVisits(butler, visitList)
def countMaskedPixels(maskedIm, maskPlane)
def _plotDefects(self, exp, visit, defects, imageType)
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
def findHotAndColdPixels(self, exp, imageType, setMask=False)
def _nPixFromDefects(defect)
def _postProcessDefectSets(self, defectList, imageDimensions, mode)
def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE')
def _writeData(self, dataRef, defects)
def __init__(self, args, kwargs)
def _plot(self, dataRef, exp, visit, nSig, defects, imageType)
def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA")
def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed)