Coverage for python/lsst/cp/pipe/defects.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['FindDefectsTask',
24 'FindDefectsTaskConfig', ]
26import numpy as np
27import os
28import warnings
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.afw.image as afwImage
33import lsst.meas.algorithms as measAlg
34import lsst.afw.math as afwMath
35import lsst.afw.detection as afwDetection
36import lsst.afw.display as afwDisplay
37from lsst.afw import cameraGeom
38from lsst.geom import Box2I, Point2I
39import lsst.daf.base as dafBase
41from lsst.ip.isr import IsrTask
42from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \
43 validateIsrConfig
46class FindDefectsTaskConfig(pexConfig.Config):
47 """Config class for defect finding"""
49 isrForFlats = pexConfig.ConfigurableField(
50 target=IsrTask,
51 doc="Task to perform instrumental signature removal",
52 )
53 isrForDarks = pexConfig.ConfigurableField(
54 target=IsrTask,
55 doc="Task to perform instrumental signature removal",
56 )
57 isrMandatoryStepsFlats = pexConfig.ListField(
58 dtype=str,
59 doc=("isr operations that must be performed for valid results when using flats."
60 " Raises if any of these are False"),
61 default=['doAssembleCcd', 'doFringe']
62 )
63 isrMandatoryStepsDarks = pexConfig.ListField(
64 dtype=str,
65 doc=("isr operations that must be performed for valid results when using darks. "
66 "Raises if any of these are False"),
67 default=['doAssembleCcd', 'doFringe']
68 )
69 isrForbiddenStepsFlats = pexConfig.ListField(
70 dtype=str,
71 doc=("isr operations that must NOT be performed for valid results when using flats."
72 " Raises if any of these are True"),
73 default=['doBrighterFatter', 'doUseOpticsTransmission',
74 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
75 )
76 isrForbiddenStepsDarks = pexConfig.ListField(
77 dtype=str,
78 doc=("isr operations that must NOT be performed for valid results when using darks."
79 " Raises if any of these are True"),
80 default=['doBrighterFatter', 'doUseOpticsTransmission',
81 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
82 )
83 isrDesirableSteps = pexConfig.ListField(
84 dtype=str,
85 doc=("isr operations that it is advisable to perform, but are not mission-critical."
86 " WARNs are logged for any of these found to be False."),
87 default=['doBias']
88 )
89 ccdKey = pexConfig.Field(
90 dtype=str,
91 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
92 default='ccd',
93 )
94 imageTypeKey = pexConfig.Field(
95 dtype=str,
96 doc="The key for the butler to use by which to check whether images are darks or flats",
97 default='imageType',
98 )
99 mode = pexConfig.ChoiceField(
100 doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?"
101 " If MASTER, a single visit number should be supplied, for which the corresponding master flat"
102 " and dark will be used. If VISITS, the list of visits will be used, treating the flats and "
103 " darks as appropriate, depending on their image types, as determined by their imageType from"
104 " config.imageTypeKey"),
105 dtype=str,
106 default="VISITS",
107 allowed={
108 "VISITS": "Calculate defects from a list of raw visits",
109 "MASTER": "Use the corresponding master calibs from the specified visit to measure defects",
110 }
111 )
112 nSigmaBright = pexConfig.Field(
113 dtype=float,
114 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be",
115 " appropriate for some LSST sensors in DM-17490."),
116 default=4.8,
117 )
118 nSigmaDark = pexConfig.Field(
119 dtype=float,
120 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be",
121 " appropriate for some LSST sensors in DM-17490."),
122 default=5.0,
123 )
124 nPixBorderUpDown = pexConfig.Field(
125 dtype=int,
126 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
127 default=7,
128 )
129 nPixBorderLeftRight = pexConfig.Field(
130 dtype=int,
131 doc="Number of pixels to exclude from left & right of image when looking for defects.",
132 default=7,
133 )
134 badOnAndOffPixelColumnThreshold = pexConfig.Field(
135 dtype=int,
136 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ",
137 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ",
138 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ",
139 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ",
140 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ",
141 "non-bad pixels in BPC, an exception to the above is made and those consecutive ",
142 "'goodPixelColumnGapThreshold' are not marked as bad."),
143 default=50,
144 )
145 goodPixelColumnGapThreshold = pexConfig.Field(
146 dtype=int,
147 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ",
148 "'badOnAndOffPixelColumnThreshold')."),
149 default=30,
150 )
151 edgesAsDefects = pexConfig.Field(
152 dtype=bool,
153 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
154 " Normal treatment is to simply exclude this region from the defect finding, such that no"
155 " defect will be located there."),
156 default=False,
157 )
158 assertSameRun = pexConfig.Field(
159 dtype=bool,
160 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or"
161 "if the run key isn't found."),
162 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
163 )
164 ignoreFilters = pexConfig.Field(
165 dtype=bool,
166 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
167 " images. Allows mixing of filters in the input flats. Set to False if you think"
168 " your defects might be chromatic and want to have registry support for varying"
169 " defects with respect to filter."),
170 default=True,
171 )
172 nullFilterName = pexConfig.Field(
173 dtype=str,
174 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
175 default="NONE",
176 )
177 combinationMode = pexConfig.ChoiceField(
178 doc="Which types of defects to identify",
179 dtype=str,
180 default="FRACTION",
181 allowed={
182 "AND": "Logical AND the pixels found in each visit to form set ",
183 "OR": "Logical OR the pixels found in each visit to form set ",
184 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
185 }
186 )
187 combinationFraction = pexConfig.RangeField(
188 dtype=float,
189 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
190 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
191 " mode AND to require pixel to appear in all images."),
192 default=0.7,
193 min=0,
194 max=1,
195 )
196 makePlots = pexConfig.Field(
197 dtype=bool,
198 doc=("Plot histograms for each visit for each amp (one plot per detector) and the final"
199 " defects overlaid on the sensor."),
200 default=False,
201 )
202 writeAs = pexConfig.ChoiceField(
203 doc="Write the output file as ASCII or FITS table",
204 dtype=str,
205 default="FITS",
206 allowed={
207 "ASCII": "Write the output as an ASCII file",
208 "FITS": "Write the output as an FITS table",
209 "BOTH": "Write the output as both a FITS table and an ASCII file",
210 }
211 )
214class FindDefectsTask(pipeBase.CmdLineTask):
215 """Task for finding defects in sensors.
217 The task has two modes of operation, defect finding in raws and in
218 master calibrations, which work as follows.
220 Master calib defect finding
221 ----------------------------
223 A single visit number is supplied, for which the corresponding flat & dark
224 will be used. This is because, at present at least, there is no way to pass
225 a calibration exposure ID from the command line to a command line task.
227 The task retrieves the corresponding dark and flat exposures for the
228 supplied visit. If a flat is available the task will (be able to) look
229 for both bright and dark defects. If only a dark is found then only bright
230 defects will be sought.
232 All pixels above/below the specified nSigma which lie with the specified
233 borders for flats/darks are identified as defects.
235 Raw visit defect finding
236 ------------------------
238 A list of exposure IDs are supplied for defect finding. The task will
239 detect bright pixels in the dark frames, if supplied, and bright & dark
240 pixels in the flats, if supplied, i.e. if you only supply darks you will
241 only be given bright defects. This is done automatically from the imageType
242 of the exposure, so the input exposure list can be a mix.
244 As with the master calib detection, all pixels above/below the specified
245 nSigma which lie with the specified borders for flats/darks are identified
246 as defects. Then, a post-processing step is done to merge these detections,
247 with pixels appearing in a fraction [0..1] of the images are kept as defects
248 and those appearing below that occurrence-threshold are discarded.
249 """
251 RunnerClass = SingleVisitListTaskRunner
252 ConfigClass = FindDefectsTaskConfig
253 _DefaultName = "findDefects"
255 def __init__(self, *args, **kwargs):
256 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
257 self.makeSubtask("isrForFlats")
258 self.makeSubtask("isrForDarks")
260 validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats,
261 self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps)
262 validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks,
263 self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps)
264 self.config.validate()
265 self.config.freeze()
267 @classmethod
268 def _makeArgumentParser(cls):
269 """Augment argument parser for the FindDefectsTask."""
270 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
271 parser.add_argument("--visitList", dest="visitList", nargs="*",
272 help=("List of visits to use. Same for each detector."
273 " Uses the normal 0..10:3^234 syntax"))
274 parser.add_id_argument("--id", datasetType="newDefects",
275 ContainerClass=NonexistentDatasetTaskDataIdContainer,
276 help="The ccds to use, e.g. --id ccd=0..100")
277 return parser
279 @pipeBase.timeMethod
280 def runDataRef(self, dataRef, visitList):
281 """Run the defect finding task.
283 Find the defects, as described in the main task docstring, from a
284 dataRef and a list of visit(s).
286 Parameters
287 ----------
288 dataRef : `lsst.daf.persistence.ButlerDataRef`
289 dataRef for the detector for the visits to be fit.
290 visitList : `list` [`int`]
291 List of visits to be processed. If config.mode == 'VISITS' then the
292 list of visits is used. If config.mode == 'MASTER' then the length
293 of visitList must be one, and the corresponding master calibrations
294 are used.
296 Returns
297 -------
298 result : `lsst.pipe.base.Struct`
299 Result struct with Components:
301 - ``defects`` : `lsst.meas.algorithms.Defect`
302 The defects found by the task.
303 - ``exitStatus`` : `int`
304 The exit code.
305 """
307 detNum = dataRef.dataId[self.config.ccdKey]
308 self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum))
310 defectLists = {'dark': [], 'flat': []}
312 midTime = 0
313 filters = set()
315 if self.config.mode == 'MASTER':
316 if len(visitList) > 1:
317 raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}")
318 dataRef.dataId['expId'] = visitList[0]
320 for datasetType in defectLists.keys():
321 exp = dataRef.get(datasetType)
322 midTime += self._getMjd(exp)
323 filters.add(exp.getFilter().getName())
324 defects = self.findHotAndColdPixels(exp, datasetType)
326 msg = "Found %s defects containing %s pixels in master %s"
327 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
328 defectLists[datasetType].append(defects)
329 if self.config.makePlots:
330 self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType),
331 defects, datasetType)
332 midTime /= len(defectLists.keys())
334 elif self.config.mode == 'VISITS':
335 butler = dataRef.getButler()
337 if self.config.assertSameRun:
338 runs = self._getRunListFromVisits(butler, visitList)
339 if len(runs) != 1:
340 raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True")
342 for visit in visitList:
343 imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'expId': visit})[0]
344 imageType = imageType.lower()
345 dataRef.dataId['expId'] = visit
347 if imageType == 'flat': # note different isr tasks
348 exp = self.isrForFlats.runDataRef(dataRef).exposure
349 defects = self.findHotAndColdPixels(exp, imageType)
350 defectLists['flat'].append(defects)
351 midTime += self._getMjd(exp)
352 filters.add(exp.getFilter().getName())
354 elif imageType == 'dark':
355 exp = self.isrForDarks.runDataRef(dataRef).exposure
356 defects = self.findHotAndColdPixels(exp, imageType)
357 defectLists['dark'].append(defects)
358 midTime += self._getMjd(exp)
359 filters.add(exp.getFilter().getName())
361 else:
362 raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported")
364 msg = "Found %s defects containing %s pixels in visit %s"
365 self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit)
367 if self.config.makePlots:
368 self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType)
370 midTime /= len(visitList)
372 msg = "Combining %s defect sets from darks for detector %s"
373 self.log.info(msg, len(defectLists['dark']), detNum)
374 mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(),
375 self.config.combinationMode)
376 msg = "Combining %s defect sets from flats for detector %s"
377 self.log.info(msg, len(defectLists['flat']), detNum)
378 mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(),
379 self.config.combinationMode)
381 msg = "Combining bright and dark defect sets for detector %s"
382 self.log.info(msg, detNum)
383 brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats]
384 allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR')
386 self._writeData(dataRef, allDefects, midTime, filters)
388 self.log.info("Finished finding defects in detector %s" % detNum)
389 return pipeBase.Struct(defects=allDefects, exitStatus=0)
391 def _getNsigmaForPlot(self, imageType):
392 assert imageType in ['flat', 'dark']
393 nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark
394 return nSig
396 @staticmethod
397 def _nPixFromDefects(defect):
398 """Count the number of pixels in a defect object."""
399 nPix = 0
400 for d in defect:
401 nPix += d.getBBox().getArea()
402 return nPix
404 def _writeData(self, dataRef, defects, midTime, filters):
405 """Write the data out to the defect file.
407 Parameters
408 ----------
409 dataRef : `lsst.daf.persistence.ButlerDataRef`
410 dataRef for the detector for defects to be written.
411 defects : `lsst.meas.algorithms.Defect`
412 The defects to be written.
413 """
414 date = dafBase.DateTime(midTime, dafBase.DateTime.MJD).toPython().isoformat()
416 detName = self._getDetectorName(dataRef)
417 instrumentName = self._getInstrumentName(dataRef)
418 detNum = self._getDetectorNumber(dataRef)
419 if not self.config.ignoreFilters:
420 filt = self._filterSetToFilterString(filters)
421 else:
422 filt = self.config.nullFilterName
424 CALIB_ID = f"detectorName={detName} detector={detNum} calibDate={date} ccd={detNum} filter={filt}"
425 try:
426 raftName = detName.split("_")[0]
427 CALIB_ID += f" raftName={raftName}"
428 except Exception:
429 pass
431 now = dafBase.DateTime.now().toPython()
432 mdOriginal = defects.getMetadata()
433 mdSupplemental = {"INSTRUME": instrumentName,
434 "DETECTOR": dataRef.dataId['detector'],
435 "CALIBDATE": date,
436 "CALIB_ID": CALIB_ID,
437 "CALIB_CREATION_DATE": now.date().isoformat(),
438 "CALIB_CREATION_TIME": now.time().isoformat()}
440 mdOriginal.update(mdSupplemental)
442 # TODO: DM-23508 sort out the butler abuse from here-on down in Gen3
443 # defects should simply be butler.put()
444 templateFilename = dataRef.getUri(write=True) # does not guarantee that full path exists
445 baseDirName = os.path.dirname(templateFilename)
446 # ingest curated calibs demands detectorName is lowercase
447 dirName = os.path.join(baseDirName, instrumentName, "defects", detName.lower())
448 if not os.path.exists(dirName):
449 os.makedirs(dirName)
451 date += ".fits"
452 filename = os.path.join(dirName, date)
454 msg = "Writing defects to %s in format: %s"
455 self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs)
456 if self.config.writeAs in ['FITS', 'BOTH']:
457 defects.writeFits(filename)
458 if self.config.writeAs in ['ASCII', 'BOTH']:
459 wroteTo = defects.writeText(filename)
460 assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0])
461 return
463 @staticmethod
464 def _filterSetToFilterString(filters):
465 return "~".join([f for f in filters])
467 @staticmethod
468 def _getDetectorNumber(dataRef):
469 dataRefDetNum = dataRef.dataId['detector']
470 camera = dataRef.get('camera')
471 detectorDetNum = camera[dataRef.dataId['detector']].getId()
472 assert dataRefDetNum == detectorDetNum
473 return dataRefDetNum
475 @staticmethod
476 def _getInstrumentName(dataRef):
477 camera = dataRef.get('camera')
478 return camera.getName()
480 @staticmethod
481 def _getDetectorName(dataRef):
482 camera = dataRef.get('camera')
483 return camera[dataRef.dataId['detector']].getName()
485 @staticmethod
486 def _getMjd(exp, timescale=dafBase.DateTime.UTC):
487 vi = exp.getInfo().getVisitInfo()
488 dateObs = vi.getDate()
489 mjd = dateObs.get(dafBase.DateTime.MJD)
490 return mjd
492 @staticmethod
493 def _getRunListFromVisits(butler, visitList):
494 """Return the set of runs for the visits in visitList."""
495 runs = set()
496 for visit in visitList:
497 runs.add(butler.queryMetadata('raw', 'run', dataId={'expId': visit})[0])
498 return runs
500 def _postProcessDefectSets(self, defectList, imageDimensions, mode):
501 """Combine a list of defects to make a single defect object.
503 AND, OR or use percentage of visits in which defects appear
504 depending on config.
506 Parameters
507 ----------
508 defectList : `list` [`lsst.meas.algorithms.Defect`]
509 The lList of defects to merge.
510 imageDimensions : `tuple` [`int`]
511 The size of the image.
512 mode : `str`
513 The combination mode to use, either 'AND', 'OR' or 'FRACTION'
515 Returns
516 -------
517 defects : `lsst.meas.algorithms.Defect`
518 The defect set resulting from the merge.
519 """
520 # so that empty lists can be passed in for input data
521 # where only flats or darks are supplied
522 if defectList == []:
523 return []
525 if len(defectList) == 1: # single input - no merging to do
526 return defectList[0]
528 sumImage = afwImage.MaskedImageF(imageDimensions)
529 for defects in defectList:
530 for defect in defects:
531 sumImage.image[defect.getBBox()] += 1
532 sumImage /= len(defectList)
534 nDetected = len(np.where(sumImage.image.array > 0)[0])
535 self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected)
537 if mode == 'OR': # must appear in any
538 indices = np.where(sumImage.image.array > 0)
539 else:
540 if mode == 'AND': # must appear in all
541 threshold = 1
542 elif mode == 'FRACTION':
543 threshold = self.config.combinationFraction
544 else:
545 raise RuntimeError(f"Got unsupported combinationMode {mode}")
546 indices = np.where(sumImage.image.array >= threshold)
548 BADBIT = sumImage.mask.getPlaneBitMask('BAD')
549 sumImage.mask.array[indices] |= BADBIT
551 self.log.info("Post-merge %s pixels marked as defects" % len(indices[0]))
553 if self.config.edgesAsDefects:
554 self.log.info("Masking edge pixels as defects in addition to previously identified defects")
555 self._setEdgeBits(sumImage, 'BAD')
557 defects = measAlg.Defects.fromMask(sumImage, 'BAD')
558 return defects
560 @staticmethod
561 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
562 """Return the number of non-bad pixels in the image."""
563 nPixels = maskedIm.mask.array.size
564 nBad = countMaskedPixels(maskedIm, badMaskString)
565 return nPixels - nBad
567 def findHotAndColdPixels(self, exp, imageType, setMask=False):
568 """Find hot and cold pixels in an image.
570 Using config-defined thresholds on a per-amp basis, mask pixels
571 that are nSigma above threshold in dark frames (hot pixels),
572 or nSigma away from the clipped mean in flats (hot & cold pixels).
574 Parameters
575 ----------
576 exp : `lsst.afw.image.exposure.Exposure`
577 The exposure in which to find defects.
578 imageType : `str`
579 The image type, either 'dark' or 'flat'.
580 setMask : `bool`
581 If true, update exp with hot and cold pixels.
582 hot: DETECTED
583 cold: DETECTED_NEGATIVE
585 Returns
586 -------
587 defects : `lsst.meas.algorithms.Defect`
588 The defects found in the image.
589 """
590 assert imageType in ['flat', 'dark']
592 self._setEdgeBits(exp)
593 maskedIm = exp.maskedImage
595 # the detection polarity for afwDetection, True for positive,
596 # False for negative, and therefore True for darks as they only have
597 # bright pixels, and both for flats, as they have bright and dark pix
598 polarities = {'dark': [True], 'flat': [True, False]}[imageType]
600 footprintList = []
602 for amp in exp.getDetector():
603 ampImg = maskedIm[amp.getBBox()].clone()
605 # crop ampImage depending on where the amp lies in the image
606 if self.config.nPixBorderLeftRight:
607 if ampImg.getX0() == 0:
608 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
609 else:
610 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
611 if self.config.nPixBorderUpDown:
612 if ampImg.getY0() == 0:
613 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
614 else:
615 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
617 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
618 continue
620 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
622 mergedSet = None
623 for polarity in polarities:
624 nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark
625 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
627 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
628 if setMask:
629 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
631 if mergedSet is None:
632 mergedSet = footprintSet
633 else:
634 mergedSet.merge(footprintSet)
636 footprintList += mergedSet.getFootprints()
638 defects = measAlg.Defects.fromFootprintList(footprintList)
639 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
641 return defects
643 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
644 """Mask blocks in a column if there are on-and-off bad pixels
646 If there's a column with on and off bad pixels, mask all the pixels in between,
647 except if there is a large enough gap of consecutive good pixels between two
648 bad pixels in the column.
650 Parameters
651 ---------
652 defects: `lsst.meas.algorithms.Defect`
653 The defects found in the image so far
655 Returns
656 ------
657 defects: `lsst.meas.algorithms.Defect`
658 If the number of bad pixels in a column is not larger or equal than
659 self.config.badPixelColumnThreshold, the iput list is returned. Otherwise,
660 the defects list returned will include boxes that mask blocks of on-and-of
661 pixels.
662 """
663 # Get the (x, y) values of each bad pixel in amp.
664 coordinates = []
665 for defect in defects:
666 bbox = defect.getBBox()
667 x0, y0 = bbox.getMinX(), bbox.getMinY()
668 deltaX0, deltaY0 = bbox.getDimensions()
669 for j in np.arange(y0, y0+deltaY0):
670 for i in np.arange(x0, x0 + deltaX0):
671 coordinates.append((i, j))
673 x, y = [], []
674 for coordinatePair in coordinates:
675 x.append(coordinatePair[0])
676 y.append(coordinatePair[1])
678 x = np.array(x)
679 y = np.array(y)
680 # Find the defects with same "x" (vertical) coordinate (column).
681 unique, counts = np.unique(x, return_counts=True)
682 multipleX = []
683 for (a, b) in zip(unique, counts):
684 if b >= self.config.badOnAndOffPixelColumnThreshold:
685 multipleX.append(a)
686 if len(multipleX) != 0:
687 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
689 return defects
691 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
692 """Mask blocks in a column if number of on-and-off bad pixels is above threshold.
694 This function is called if the number of on-and-off bad pixels in a column
695 is larger or equal than self.config.badOnAndOffPixelColumnThreshold.
697 Parameters
698 ---------
699 x: list
700 Lower left x coordinate of defect box. x coordinate is along the short axis if amp.
702 y: list
703 Lower left y coordinate of defect box. x coordinate is along the long axis if amp.
705 multipleX: list
706 List of x coordinates in amp. with multiple bad pixels (i.e., columns with defects).
708 defects: `lsst.meas.algorithms.Defect`
709 The defcts found in the image so far
711 Returns
712 -------
713 defects: `lsst.meas.algorithms.Defect`
714 The defects list returned that will include boxes that mask blocks
715 of on-and-of pixels.
716 """
717 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
718 for x0 in multipleX:
719 index = np.where(x == x0)
720 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
721 minY, maxY = np.min(multipleY), np.max(multipleY)
722 # Next few lines: don't mask pixels in column if gap of good pixels between
723 # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'.
724 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
725 if len(diffIndex) != 0:
726 limits = [minY] # put the minimum first
727 for gapIndex in diffIndex:
728 limits.append(multipleY[gapIndex])
729 limits.append(multipleY[gapIndex+1])
730 limits.append(maxY) # maximum last
731 assert len(limits)%2 == 0, 'limits is even by design, but check anyways'
732 for i in np.arange(0, len(limits)-1, 2):
733 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
734 if s not in defects:
735 defects.append(s)
736 else: # No gap is large enough
737 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
738 if s not in defects:
739 defects.append(s)
740 return defects
742 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
743 """Set edge bits on an exposure or maskedImage.
745 Raises
746 ------
747 TypeError
748 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
749 """
750 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
751 mi = exposureOrMaskedImage.maskedImage
752 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
753 mi = exposureOrMaskedImage
754 else:
755 t = type(exposureOrMaskedImage)
756 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
758 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
759 if self.config.nPixBorderLeftRight:
760 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
761 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
762 if self.config.nPixBorderUpDown:
763 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
764 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
766 def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover
767 """Plot the defects and pixel histograms.
769 Parameters
770 ----------
771 dataRef : `lsst.daf.persistence.ButlerDataRef`
772 dataRef for the detector.
773 exp : `lsst.afw.image.exposure.Exposure`
774 The exposure in which the defects were found.
775 visit : `int`
776 The visit number.
777 nSig : `float`
778 The number of sigma used for detection
779 defects : `lsst.meas.algorithms.Defect`
780 The defects to plot.
781 imageType : `str`
782 The type of image, either 'dark' or 'flat'.
784 Currently only for LSST sensors. Plots are written to the path
785 given by the butler for the ``cpPipePlotRoot`` dataset type.
786 """
787 import matplotlib.pyplot as plt
788 from matplotlib.backends.backend_pdf import PdfPages
790 afwDisplay.setDefaultBackend("matplotlib")
791 plt.interactive(False) # seems to need reasserting here
793 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
794 if not os.path.exists(dirname):
795 os.makedirs(dirname)
797 detNum = exp.getDetector().getId()
798 nAmps = len(exp.getDetector())
800 if self.config.mode == "MASTER":
801 filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf"
802 elif self.config.mode == "VISITS":
803 filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf"
805 filenameFull = os.path.join(dirname, filename)
807 with warnings.catch_warnings():
808 msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure."
809 warnings.filterwarnings("ignore", message=msg)
810 with PdfPages(filenameFull) as pdfPages:
811 if nAmps == 16:
812 self._plotAmpHistogram(dataRef, exp, visit, nSig)
813 pdfPages.savefig()
815 self._plotDefects(exp, visit, defects, imageType)
816 pdfPages.savefig()
817 self.log.info("Wrote plot(s) to %s" % filenameFull)
819 def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover
820 """Plot the defects found by the task.
822 Parameters
823 ----------
824 exp : `lsst.afw.image.exposure.Exposure`
825 The exposure in which the defects were found.
826 visit : `int`
827 The visit number.
828 defects : `lsst.meas.algorithms.Defect`
829 The defects to plot.
830 imageType : `str`
831 The type of image, either 'dark' or 'flat'.
832 """
833 expCopy = exp.clone() # we mess with the copy later, so make a clone
834 del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes
835 maskedIm = expCopy.maskedImage
837 defects.maskPixels(expCopy.maskedImage, "BAD")
838 detector = expCopy.getDetector()
840 disp = afwDisplay.Display(0, reopenPlot=True, dpi=200)
842 if imageType == "flat": # set each amp image to have a mean of 1.00
843 for amp in detector:
844 ampIm = maskedIm.image[amp.getBBox()]
845 ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1
847 mpDict = maskedIm.mask.getMaskPlaneDict()
848 for plane in mpDict.keys():
849 if plane in ['BAD']:
850 continue
851 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
853 disp.scale('asinh', 'zscale')
854 disp.setMaskTransparency(80)
855 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
857 disp.setImageColormap('gray')
858 title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}"
859 f", Type: {imageType}, visit: {visit}")
860 disp.mtv(maskedIm, title=title)
862 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
864 def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover
865 """
866 Make a histogram of the distribution of pixel values for each amp.
868 The main image data histogram is plotted in blue. Edge pixels,
869 if masked, are in red. Note that masked edge pixels do not contribute
870 to the underflow and overflow numbers.
872 Note that this currently only supports the 16-amp LSST detectors.
874 Parameters
875 ----------
876 dataRef : `lsst.daf.persistence.ButlerDataRef`
877 dataRef for the detector.
878 exp : `lsst.afw.image.exposure.Exposure`
879 The exposure in which the defects were found.
880 visit : `int`
881 The visit number.
882 nSigmaUsed : `float`
883 The number of sigma used for detection
884 """
885 import matplotlib.pyplot as plt
887 detector = exp.getDetector()
889 if len(detector) != 16:
890 raise RuntimeError("Plotting currently only supported for 16 amp detectors")
891 fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
893 expTime = exp.getInfo().getVisitInfo().getExposureTime()
895 for (amp, a) in zip(reversed(detector), ax.flatten()):
896 mi = exp.maskedImage[amp.getBBox()]
898 # normalize by expTime as we plot in ADU/s and don't always work with master calibs
899 mi.image.array /= expTime
900 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
901 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
903 # Get array of pixels
904 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
905 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
906 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
908 thrUpper = mean + nSigmaUsed*sigma
909 thrLower = mean - nSigmaUsed*sigma
911 nRight = len(imgData[imgData > thrUpper])
912 nLeft = len(imgData[imgData < thrLower])
914 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
915 leftEdge = mean - nsig * nSigmaUsed*sigma
916 rightEdge = mean + nsig * nSigmaUsed*sigma
917 nbins = np.linspace(leftEdge, rightEdge, 1000)
918 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red')
919 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue')
921 # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram
922 nOverflow = len(imgData[imgData > rightEdge])
923 nUnderflow = len(imgData[imgData < leftEdge])
925 # Put v-lines and textboxes in
926 a.axvline(thrUpper, c='k')
927 a.axvline(thrLower, c='k')
928 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
929 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
930 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
931 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
933 # set axis limits and scales
934 a.set_ylim([1., 1.7*np.max(y)])
935 lPlot, rPlot = a.get_xlim()
936 a.set_xlim(np.array([lPlot, rPlot]))
937 a.set_yscale('log')
938 a.set_xlabel("ADU/s")
940 return