Coverage for python/lsst/cp/pipe/defects.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
23__all__ = ['FindDefectsTask',
24 'FindDefectsTaskConfig', ]
26import numpy as np
27import os
28import warnings
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.afw.image as afwImage
33import lsst.meas.algorithms as measAlg
34import lsst.afw.math as afwMath
35import lsst.afw.detection as afwDetection
36import lsst.afw.display as afwDisplay
37from lsst.afw import cameraGeom
38from lsst.geom import Box2I, Point2I
39import lsst.daf.base as dafBase
41from lsst.ip.isr import IsrTask
42from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \
43 validateIsrConfig
46class FindDefectsTaskConfig(pexConfig.Config):
47 """Config class for defect finding"""
49 isrForFlats = pexConfig.ConfigurableField(
50 target=IsrTask,
51 doc="Task to perform instrumental signature removal",
52 )
53 isrForDarks = pexConfig.ConfigurableField(
54 target=IsrTask,
55 doc="Task to perform instrumental signature removal",
56 )
57 isrMandatoryStepsFlats = pexConfig.ListField(
58 dtype=str,
59 doc=("isr operations that must be performed for valid results when using flats."
60 " Raises if any of these are False"),
61 default=['doAssembleCcd', 'doFringe']
62 )
63 isrMandatoryStepsDarks = pexConfig.ListField(
64 dtype=str,
65 doc=("isr operations that must be performed for valid results when using darks. "
66 "Raises if any of these are False"),
67 default=['doAssembleCcd', 'doFringe']
68 )
69 isrForbiddenStepsFlats = pexConfig.ListField(
70 dtype=str,
71 doc=("isr operations that must NOT be performed for valid results when using flats."
72 " Raises if any of these are True"),
73 default=['doBrighterFatter', 'doUseOpticsTransmission',
74 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
75 )
76 isrForbiddenStepsDarks = pexConfig.ListField(
77 dtype=str,
78 doc=("isr operations that must NOT be performed for valid results when using darks."
79 " Raises if any of these are True"),
80 default=['doBrighterFatter', 'doUseOpticsTransmission',
81 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
82 )
83 isrDesirableSteps = pexConfig.ListField(
84 dtype=str,
85 doc=("isr operations that it is advisable to perform, but are not mission-critical."
86 " WARNs are logged for any of these found to be False."),
87 default=['doBias']
88 )
89 ccdKey = pexConfig.Field(
90 dtype=str,
91 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
92 default='ccd',
93 )
94 imageTypeKey = pexConfig.Field(
95 dtype=str,
96 doc="The key for the butler to use by which to check whether images are darks or flats",
97 default='imageType',
98 )
99 mode = pexConfig.ChoiceField(
100 doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?"
101 " If MASTER, a single visit number should be supplied, for which the corresponding master flat"
102 " and dark will be used. If VISITS, the list of visits will be used, treating the flats and "
103 " darks as appropriate, depending on their image types, as determined by their imageType from"
104 " config.imageTypeKey"),
105 dtype=str,
106 default="VISITS",
107 allowed={
108 "VISITS": "Calculate defects from a list of raw visits",
109 "MASTER": "Use the corresponding master calibs from the specified visit to measure defects",
110 }
111 )
112 nSigmaBright = pexConfig.Field(
113 dtype=float,
114 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be",
115 " appropriate for some LSST sensors in DM-17490."),
116 default=4.8,
117 )
118 nSigmaDark = pexConfig.Field(
119 dtype=float,
120 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be",
121 " appropriate for some LSST sensors in DM-17490."),
122 default=5.0,
123 )
124 nPixBorderUpDown = pexConfig.Field(
125 dtype=int,
126 doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
127 default=7,
128 )
129 nPixBorderLeftRight = pexConfig.Field(
130 dtype=int,
131 doc="Number of pixels to exclude from left & right of image when looking for defects.",
132 default=7,
133 )
134 badOnAndOffPixelColumnThreshold = pexConfig.Field(
135 dtype=int,
136 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ",
137 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ",
138 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ",
139 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ",
140 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ",
141 "non-bad pixels in BPC, an exception to the above is made and those consecutive ",
142 "'goodPixelColumnGapThreshold' are not marked as bad."),
143 default=50,
144 )
145 goodPixelColumnGapThreshold = pexConfig.Field(
146 dtype=int,
147 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ",
148 "'badOnAndOffPixelColumnThreshold')."),
149 default=30,
150 )
151 edgesAsDefects = pexConfig.Field(
152 dtype=bool,
153 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
154 " Normal treatment is to simply exclude this region from the defect finding, such that no"
155 " defect will be located there."),
156 default=False,
157 )
158 assertSameRun = pexConfig.Field(
159 dtype=bool,
160 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or"
161 "if the run key isn't found."),
162 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
163 )
164 ignoreFilters = pexConfig.Field(
165 dtype=bool,
166 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
167 " images. Allows mixing of filters in the input flats. Set to False if you think"
168 " your defects might be chromatic and want to have registry support for varying"
169 " defects with respect to filter."),
170 default=True,
171 )
172 nullFilterName = pexConfig.Field(
173 dtype=str,
174 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
175 default="NONE",
176 )
177 combinationMode = pexConfig.ChoiceField(
178 doc="Which types of defects to identify",
179 dtype=str,
180 default="FRACTION",
181 allowed={
182 "AND": "Logical AND the pixels found in each visit to form set ",
183 "OR": "Logical OR the pixels found in each visit to form set ",
184 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
185 }
186 )
187 combinationFraction = pexConfig.RangeField(
188 dtype=float,
189 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
190 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
191 " mode AND to require pixel to appear in all images."),
192 default=0.7,
193 min=0,
194 max=1,
195 )
196 makePlots = pexConfig.Field(
197 dtype=bool,
198 doc=("Plot histograms for each visit for each amp (one plot per detector) and the final"
199 " defects overlaid on the sensor."),
200 default=False,
201 )
202 writeAs = pexConfig.ChoiceField(
203 doc="Write the output file as ASCII or FITS table",
204 dtype=str,
205 default="FITS",
206 allowed={
207 "ASCII": "Write the output as an ASCII file",
208 "FITS": "Write the output as an FITS table",
209 "BOTH": "Write the output as both a FITS table and an ASCII file",
210 }
211 )
214class FindDefectsTask(pipeBase.CmdLineTask):
215 """Task for finding defects in sensors.
217 The task has two modes of operation, defect finding in raws and in
218 master calibrations, which work as follows.
220 Master calib defect finding
221 ----------------------------
223 A single visit number is supplied, for which the corresponding flat & dark
224 will be used. This is because, at present at least, there is no way to pass
225 a calibration exposure ID from the command line to a command line task.
227 The task retrieves the corresponding dark and flat exposures for the
228 supplied visit. If a flat is available the task will (be able to) look
229 for both bright and dark defects. If only a dark is found then only bright
230 defects will be sought.
232 All pixels above/below the specified nSigma which lie with the specified
233 borders for flats/darks are identified as defects.
235 Raw visit defect finding
236 ------------------------
238 A list of exposure IDs are supplied for defect finding. The task will
239 detect bright pixels in the dark frames, if supplied, and bright & dark
240 pixels in the flats, if supplied, i.e. if you only supply darks you will
241 only be given bright defects. This is done automatically from the imageType
242 of the exposure, so the input exposure list can be a mix.
244 As with the master calib detection, all pixels above/below the specified
245 nSigma which lie with the specified borders for flats/darks are identified
246 as defects. Then, a post-processing step is done to merge these detections,
247 with pixels appearing in a fraction [0..1] of the images are kept as defects
248 and those appearing below that occurrence-threshold are discarded.
249 """
251 RunnerClass = SingleVisitListTaskRunner
252 ConfigClass = FindDefectsTaskConfig
253 _DefaultName = "findDefects"
255 def __init__(self, *args, **kwargs):
256 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
257 self.makeSubtask("isrForFlats")
258 self.makeSubtask("isrForDarks")
260 validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats,
261 self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps)
262 validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks,
263 self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps)
264 self.config.validate()
265 self.config.freeze()
267 @classmethod
268 def _makeArgumentParser(cls):
269 """Augment argument parser for the FindDefectsTask."""
270 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
271 parser.add_argument("--visitList", dest="visitList", nargs="*",
272 help=("List of visits to use. Same for each detector."
273 " Uses the normal 0..10:3^234 syntax"))
274 parser.add_id_argument("--id", datasetType="newDefects",
275 ContainerClass=NonexistentDatasetTaskDataIdContainer,
276 help="The ccds to use, e.g. --id ccd=0..100")
277 return parser
279 @pipeBase.timeMethod
280 def runDataRef(self, dataRef, visitList):
281 """Run the defect finding task.
283 Find the defects, as described in the main task docstring, from a
284 dataRef and a list of visit(s).
286 Parameters
287 ----------
288 dataRef : `lsst.daf.persistence.ButlerDataRef`
289 dataRef for the detector for the visits to be fit.
290 visitList : `list` [`int`]
291 List of visits to be processed. If config.mode == 'VISITS' then the
292 list of visits is used. If config.mode == 'MASTER' then the length
293 of visitList must be one, and the corresponding master calibrations
294 are used.
296 Returns
297 -------
298 result : `lsst.pipe.base.Struct`
299 Result struct with Components:
301 - ``defects`` : `lsst.meas.algorithms.Defect`
302 The defects found by the task.
303 - ``exitStatus`` : `int`
304 The exit code.
305 """
307 detNum = dataRef.dataId[self.config.ccdKey]
308 self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum))
310 defectLists = {'dark': [], 'flat': []}
312 midTime = 0
313 filters = set()
315 if self.config.mode == 'MASTER':
316 if len(visitList) > 1:
317 raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}")
318 dataRef.dataId['expId'] = visitList[0]
320 for datasetType in defectLists.keys():
321 exp = dataRef.get(datasetType)
322 midTime += self._getMjd(exp)
323 filters.add(exp.getFilter().getName())
324 defects = self.findHotAndColdPixels(exp, datasetType)
326 msg = "Found %s defects containing %s pixels in master %s"
327 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
328 defectLists[datasetType].append(defects)
329 if self.config.makePlots:
330 self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType),
331 defects, datasetType)
332 midTime /= len(defectLists.keys())
334 elif self.config.mode == 'VISITS':
335 butler = dataRef.getButler()
337 if self.config.assertSameRun:
338 runs = self._getRunListFromVisits(butler, visitList)
339 if len(runs) != 1:
340 raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True")
342 for visit in visitList:
343 imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'expId': visit})[0]
344 imageType = imageType.lower()
345 dataRef.dataId['expId'] = visit
347 if imageType == 'flat': # note different isr tasks
348 exp = self.isrForFlats.runDataRef(dataRef).exposure
349 defects = self.findHotAndColdPixels(exp, imageType)
350 defectLists['flat'].append(defects)
351 midTime += self._getMjd(exp)
352 filters.add(exp.getFilter().getName())
354 elif imageType == 'dark':
355 exp = self.isrForDarks.runDataRef(dataRef).exposure
356 defects = self.findHotAndColdPixels(exp, imageType)
357 defectLists['dark'].append(defects)
358 midTime += self._getMjd(exp)
359 filters.add(exp.getFilter().getName())
361 else:
362 raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported")
364 msg = "Found %s defects containing %s pixels in visit %s"
365 self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit)
367 if self.config.makePlots:
368 self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType)
370 midTime /= len(visitList)
372 msg = "Combining %s defect sets from darks for detector %s"
373 self.log.info(msg, len(defectLists['dark']), detNum)
374 mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(),
375 self.config.combinationMode)
376 msg = "Combining %s defect sets from flats for detector %s"
377 self.log.info(msg, len(defectLists['flat']), detNum)
378 mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(),
379 self.config.combinationMode)
381 msg = "Combining bright and dark defect sets for detector %s"
382 self.log.info(msg, detNum)
383 brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats]
384 allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR')
386 self._writeData(dataRef, allDefects, midTime, filters)
388 self.log.info("Finished finding defects in detector %s" % detNum)
389 return pipeBase.Struct(defects=allDefects, exitStatus=0)
391 def _getNsigmaForPlot(self, imageType):
392 assert imageType in ['flat', 'dark']
393 nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark
394 return nSig
396 @staticmethod
397 def _nPixFromDefects(defect):
398 """Count the number of pixels in a defect object."""
399 nPix = 0
400 for d in defect:
401 nPix += d.getBBox().getArea()
402 return nPix
404 def _writeData(self, dataRef, defects, midTime, filters):
405 """Write the data out to the defect file.
407 Parameters
408 ----------
409 dataRef : `lsst.daf.persistence.ButlerDataRef`
410 dataRef for the detector for defects to be written.
411 defects : `lsst.meas.algorithms.Defect`
412 The defects to be written.
413 """
414 date = dafBase.DateTime(midTime, dafBase.DateTime.MJD).toPython().isoformat()
416 detName = self._getDetectorNameShort(dataRef)
417 instrumentName = self._getInstrumentName(dataRef)
418 detNum = self._getDetectorNumber(dataRef)
419 if not self.config.ignoreFilters:
420 filt = self._filterSetToFilterString(filters)
421 else:
422 filt = self.config.nullFilterName
424 CALIB_ID = f"detectorName={detName} detector={detNum} calibDate={date} ccd={detNum} filter={filt}"
425 try:
426 raftName = self._getRaftName(dataRef)
427 CALIB_ID += f" raftName={raftName}"
428 except Exception:
429 pass
431 now = dafBase.DateTime.now().toPython()
432 mdOriginal = defects.getMetadata()
433 mdSupplemental = {"INSTRUME": instrumentName,
434 "DETECTOR": dataRef.dataId['detector'],
435 "CALIBDATE": date,
436 "CALIB_ID": CALIB_ID,
437 "CALIB_CREATION_DATE": now.date().isoformat(),
438 "CALIB_CREATION_TIME": now.time().isoformat()}
440 mdOriginal.update(mdSupplemental)
442 # TODO: DM-23508 sort out the butler abuse from here-on down in Gen3
443 # defects should simply be butler.put()
444 templateFilename = dataRef.getUri(write=True) # does not guarantee that full path exists
445 baseDirName = os.path.dirname(templateFilename)
446 # ingest curated calibs demands detectorName is lowercase
447 detNameFull = self._getDetectorNameFull(dataRef)
448 dirName = os.path.join(baseDirName, instrumentName, "defects", detNameFull.lower())
449 if not os.path.exists(dirName):
450 os.makedirs(dirName)
452 date += ".fits"
453 filename = os.path.join(dirName, date)
455 msg = "Writing defects to %s in format: %s"
456 self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs)
457 if self.config.writeAs in ['FITS', 'BOTH']:
458 defects.writeFits(filename)
459 if self.config.writeAs in ['ASCII', 'BOTH']:
460 wroteTo = defects.writeText(filename)
461 assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0])
462 return
464 @staticmethod
465 def _filterSetToFilterString(filters):
466 return "~".join([f for f in filters])
468 @staticmethod
469 def _getDetectorNumber(dataRef):
470 """The detector's integer identifier."""
471 dataRefDetNum = dataRef.dataId['detector']
472 camera = dataRef.get('camera')
473 detectorDetNum = camera[dataRef.dataId['detector']].getId()
474 assert dataRefDetNum == detectorDetNum
475 return dataRefDetNum
477 @staticmethod
478 def _getInstrumentName(dataRef):
479 camera = dataRef.get('camera')
480 return camera.getName()
482 @staticmethod
483 def _getDetectorNameFull(dataRef):
484 """The detector's self-reported full name, e.g. R12_S01."""
485 camera = dataRef.get('camera')
486 return camera[dataRef.dataId['detector']].getName()
488 @staticmethod
489 def _getDetectorNameShort(dataRef):
490 """The detectorName per the butler, e.g. slot name, e.g. S12."""
491 butler = dataRef.getButler()
492 detectorName = butler.queryMetadata('raw', ['detectorName'], dataRef.dataId)[0]
493 return detectorName
495 @staticmethod
496 def _getRaftName(dataRef):
497 """The detectorName per the butler, e.g. slot name, e.g. S12."""
498 butler = dataRef.getButler()
499 raftName = butler.queryMetadata('raw', ['raftName'], dataRef.dataId)[0]
500 return raftName
502 @staticmethod
503 def _getMjd(exp, timescale=dafBase.DateTime.UTC):
504 vi = exp.getInfo().getVisitInfo()
505 dateObs = vi.getDate()
506 mjd = dateObs.get(dafBase.DateTime.MJD)
507 return mjd
509 @staticmethod
510 def _getRunListFromVisits(butler, visitList):
511 """Return the set of runs for the visits in visitList."""
512 runs = set()
513 for visit in visitList:
514 runs.add(butler.queryMetadata('raw', 'run', dataId={'expId': visit})[0])
515 return runs
517 def _postProcessDefectSets(self, defectList, imageDimensions, mode):
518 """Combine a list of defects to make a single defect object.
520 AND, OR or use percentage of visits in which defects appear
521 depending on config.
523 Parameters
524 ----------
525 defectList : `list` [`lsst.meas.algorithms.Defect`]
526 The lList of defects to merge.
527 imageDimensions : `tuple` [`int`]
528 The size of the image.
529 mode : `str`
530 The combination mode to use, either 'AND', 'OR' or 'FRACTION'
532 Returns
533 -------
534 defects : `lsst.meas.algorithms.Defect`
535 The defect set resulting from the merge.
536 """
537 # so that empty lists can be passed in for input data
538 # where only flats or darks are supplied
539 if defectList == []:
540 return []
542 if len(defectList) == 1: # single input - no merging to do
543 return defectList[0]
545 sumImage = afwImage.MaskedImageF(imageDimensions)
546 for defects in defectList:
547 for defect in defects:
548 sumImage.image[defect.getBBox()] += 1
549 sumImage /= len(defectList)
551 nDetected = len(np.where(sumImage.image.array > 0)[0])
552 self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected)
554 if mode == 'OR': # must appear in any
555 indices = np.where(sumImage.image.array > 0)
556 else:
557 if mode == 'AND': # must appear in all
558 threshold = 1
559 elif mode == 'FRACTION':
560 threshold = self.config.combinationFraction
561 else:
562 raise RuntimeError(f"Got unsupported combinationMode {mode}")
563 indices = np.where(sumImage.image.array >= threshold)
565 BADBIT = sumImage.mask.getPlaneBitMask('BAD')
566 sumImage.mask.array[indices] |= BADBIT
568 self.log.info("Post-merge %s pixels marked as defects" % len(indices[0]))
570 if self.config.edgesAsDefects:
571 self.log.info("Masking edge pixels as defects in addition to previously identified defects")
572 self._setEdgeBits(sumImage, 'BAD')
574 defects = measAlg.Defects.fromMask(sumImage, 'BAD')
575 return defects
577 @staticmethod
578 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
579 """Return the number of non-bad pixels in the image."""
580 nPixels = maskedIm.mask.array.size
581 nBad = countMaskedPixels(maskedIm, badMaskString)
582 return nPixels - nBad
584 def findHotAndColdPixels(self, exp, imageType, setMask=False):
585 """Find hot and cold pixels in an image.
587 Using config-defined thresholds on a per-amp basis, mask pixels
588 that are nSigma above threshold in dark frames (hot pixels),
589 or nSigma away from the clipped mean in flats (hot & cold pixels).
591 Parameters
592 ----------
593 exp : `lsst.afw.image.exposure.Exposure`
594 The exposure in which to find defects.
595 imageType : `str`
596 The image type, either 'dark' or 'flat'.
597 setMask : `bool`
598 If true, update exp with hot and cold pixels.
599 hot: DETECTED
600 cold: DETECTED_NEGATIVE
602 Returns
603 -------
604 defects : `lsst.meas.algorithms.Defect`
605 The defects found in the image.
606 """
607 assert imageType in ['flat', 'dark']
609 self._setEdgeBits(exp)
610 maskedIm = exp.maskedImage
612 # the detection polarity for afwDetection, True for positive,
613 # False for negative, and therefore True for darks as they only have
614 # bright pixels, and both for flats, as they have bright and dark pix
615 polarities = {'dark': [True], 'flat': [True, False]}[imageType]
617 footprintList = []
619 for amp in exp.getDetector():
620 ampImg = maskedIm[amp.getBBox()].clone()
622 # crop ampImage depending on where the amp lies in the image
623 if self.config.nPixBorderLeftRight:
624 if ampImg.getX0() == 0:
625 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
626 else:
627 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
628 if self.config.nPixBorderUpDown:
629 if ampImg.getY0() == 0:
630 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
631 else:
632 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
634 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
635 continue
637 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
639 mergedSet = None
640 for polarity in polarities:
641 nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark
642 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
644 footprintSet = afwDetection.FootprintSet(ampImg, threshold)
645 if setMask:
646 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
648 if mergedSet is None:
649 mergedSet = footprintSet
650 else:
651 mergedSet.merge(footprintSet)
653 footprintList += mergedSet.getFootprints()
655 defects = measAlg.Defects.fromFootprintList(footprintList)
656 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
658 return defects
660 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects):
661 """Mask blocks in a column if there are on-and-off bad pixels
663 If there's a column with on and off bad pixels, mask all the pixels in between,
664 except if there is a large enough gap of consecutive good pixels between two
665 bad pixels in the column.
667 Parameters
668 ---------
669 defects: `lsst.meas.algorithms.Defect`
670 The defects found in the image so far
672 Returns
673 ------
674 defects: `lsst.meas.algorithms.Defect`
675 If the number of bad pixels in a column is not larger or equal than
676 self.config.badPixelColumnThreshold, the iput list is returned. Otherwise,
677 the defects list returned will include boxes that mask blocks of on-and-of
678 pixels.
679 """
680 # Get the (x, y) values of each bad pixel in amp.
681 coordinates = []
682 for defect in defects:
683 bbox = defect.getBBox()
684 x0, y0 = bbox.getMinX(), bbox.getMinY()
685 deltaX0, deltaY0 = bbox.getDimensions()
686 for j in np.arange(y0, y0+deltaY0):
687 for i in np.arange(x0, x0 + deltaX0):
688 coordinates.append((i, j))
690 x, y = [], []
691 for coordinatePair in coordinates:
692 x.append(coordinatePair[0])
693 y.append(coordinatePair[1])
695 x = np.array(x)
696 y = np.array(y)
697 # Find the defects with same "x" (vertical) coordinate (column).
698 unique, counts = np.unique(x, return_counts=True)
699 multipleX = []
700 for (a, b) in zip(unique, counts):
701 if b >= self.config.badOnAndOffPixelColumnThreshold:
702 multipleX.append(a)
703 if len(multipleX) != 0:
704 defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
706 return defects
708 def _markBlocksInBadColumn(self, x, y, multipleX, defects):
709 """Mask blocks in a column if number of on-and-off bad pixels is above threshold.
711 This function is called if the number of on-and-off bad pixels in a column
712 is larger or equal than self.config.badOnAndOffPixelColumnThreshold.
714 Parameters
715 ---------
716 x: list
717 Lower left x coordinate of defect box. x coordinate is along the short axis if amp.
719 y: list
720 Lower left y coordinate of defect box. x coordinate is along the long axis if amp.
722 multipleX: list
723 List of x coordinates in amp. with multiple bad pixels (i.e., columns with defects).
725 defects: `lsst.meas.algorithms.Defect`
726 The defcts found in the image so far
728 Returns
729 -------
730 defects: `lsst.meas.algorithms.Defect`
731 The defects list returned that will include boxes that mask blocks
732 of on-and-of pixels.
733 """
734 with defects.bulk_update():
735 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
736 for x0 in multipleX:
737 index = np.where(x == x0)
738 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
739 minY, maxY = np.min(multipleY), np.max(multipleY)
740 # Next few lines: don't mask pixels in column if gap of good pixels between
741 # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'.
742 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
743 if len(diffIndex) != 0:
744 limits = [minY] # put the minimum first
745 for gapIndex in diffIndex:
746 limits.append(multipleY[gapIndex])
747 limits.append(multipleY[gapIndex+1])
748 limits.append(maxY) # maximum last
749 assert len(limits)%2 == 0, 'limits is even by design, but check anyways'
750 for i in np.arange(0, len(limits)-1, 2):
751 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
752 defects.append(s)
753 else: # No gap is large enough
754 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
755 defects.append(s)
756 return defects
758 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
759 """Set edge bits on an exposure or maskedImage.
761 Raises
762 ------
763 TypeError
764 Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
765 """
766 if isinstance(exposureOrMaskedImage, afwImage.Exposure):
767 mi = exposureOrMaskedImage.maskedImage
768 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
769 mi = exposureOrMaskedImage
770 else:
771 t = type(exposureOrMaskedImage)
772 raise TypeError(f"Function supports exposure or maskedImage but not {t}")
774 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
775 if self.config.nPixBorderLeftRight:
776 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
777 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
778 if self.config.nPixBorderUpDown:
779 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
780 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
782 def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover
783 """Plot the defects and pixel histograms.
785 Parameters
786 ----------
787 dataRef : `lsst.daf.persistence.ButlerDataRef`
788 dataRef for the detector.
789 exp : `lsst.afw.image.exposure.Exposure`
790 The exposure in which the defects were found.
791 visit : `int`
792 The visit number.
793 nSig : `float`
794 The number of sigma used for detection
795 defects : `lsst.meas.algorithms.Defect`
796 The defects to plot.
797 imageType : `str`
798 The type of image, either 'dark' or 'flat'.
800 Currently only for LSST sensors. Plots are written to the path
801 given by the butler for the ``cpPipePlotRoot`` dataset type.
802 """
803 import matplotlib.pyplot as plt
804 from matplotlib.backends.backend_pdf import PdfPages
806 afwDisplay.setDefaultBackend("matplotlib")
807 plt.interactive(False) # seems to need reasserting here
809 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
810 if not os.path.exists(dirname):
811 os.makedirs(dirname)
813 detNum = exp.getDetector().getId()
814 nAmps = len(exp.getDetector())
816 if self.config.mode == "MASTER":
817 filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf"
818 elif self.config.mode == "VISITS":
819 filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf"
821 filenameFull = os.path.join(dirname, filename)
823 with warnings.catch_warnings():
824 msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure."
825 warnings.filterwarnings("ignore", message=msg)
826 with PdfPages(filenameFull) as pdfPages:
827 if nAmps == 16:
828 self._plotAmpHistogram(dataRef, exp, visit, nSig)
829 pdfPages.savefig()
831 self._plotDefects(exp, visit, defects, imageType)
832 pdfPages.savefig()
833 self.log.info("Wrote plot(s) to %s" % filenameFull)
835 def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover
836 """Plot the defects found by the task.
838 Parameters
839 ----------
840 exp : `lsst.afw.image.exposure.Exposure`
841 The exposure in which the defects were found.
842 visit : `int`
843 The visit number.
844 defects : `lsst.meas.algorithms.Defect`
845 The defects to plot.
846 imageType : `str`
847 The type of image, either 'dark' or 'flat'.
848 """
849 expCopy = exp.clone() # we mess with the copy later, so make a clone
850 del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes
851 maskedIm = expCopy.maskedImage
853 defects.maskPixels(expCopy.maskedImage, "BAD")
854 detector = expCopy.getDetector()
856 disp = afwDisplay.Display(0, reopenPlot=True, dpi=200)
858 if imageType == "flat": # set each amp image to have a mean of 1.00
859 for amp in detector:
860 ampIm = maskedIm.image[amp.getBBox()]
861 ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1
863 mpDict = maskedIm.mask.getMaskPlaneDict()
864 for plane in mpDict.keys():
865 if plane in ['BAD']:
866 continue
867 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
869 disp.scale('asinh', 'zscale')
870 disp.setMaskTransparency(80)
871 disp.setMaskPlaneColor("BAD", afwDisplay.RED)
873 disp.setImageColormap('gray')
874 title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}"
875 f", Type: {imageType}, visit: {visit}")
876 disp.mtv(maskedIm, title=title)
878 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
880 def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover
881 """
882 Make a histogram of the distribution of pixel values for each amp.
884 The main image data histogram is plotted in blue. Edge pixels,
885 if masked, are in red. Note that masked edge pixels do not contribute
886 to the underflow and overflow numbers.
888 Note that this currently only supports the 16-amp LSST detectors.
890 Parameters
891 ----------
892 dataRef : `lsst.daf.persistence.ButlerDataRef`
893 dataRef for the detector.
894 exp : `lsst.afw.image.exposure.Exposure`
895 The exposure in which the defects were found.
896 visit : `int`
897 The visit number.
898 nSigmaUsed : `float`
899 The number of sigma used for detection
900 """
901 import matplotlib.pyplot as plt
903 detector = exp.getDetector()
905 if len(detector) != 16:
906 raise RuntimeError("Plotting currently only supported for 16 amp detectors")
907 fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
909 expTime = exp.getInfo().getVisitInfo().getExposureTime()
911 for (amp, a) in zip(reversed(detector), ax.flatten()):
912 mi = exp.maskedImage[amp.getBBox()]
914 # normalize by expTime as we plot in ADU/s and don't always work with master calibs
915 mi.image.array /= expTime
916 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
917 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
919 # Get array of pixels
920 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
921 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
922 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
924 thrUpper = mean + nSigmaUsed*sigma
925 thrLower = mean - nSigmaUsed*sigma
927 nRight = len(imgData[imgData > thrUpper])
928 nLeft = len(imgData[imgData < thrLower])
930 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
931 leftEdge = mean - nsig * nSigmaUsed*sigma
932 rightEdge = mean + nsig * nSigmaUsed*sigma
933 nbins = np.linspace(leftEdge, rightEdge, 1000)
934 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red')
935 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue')
937 # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram
938 nOverflow = len(imgData[imgData > rightEdge])
939 nUnderflow = len(imgData[imgData < leftEdge])
941 # Put v-lines and textboxes in
942 a.axvline(thrUpper, c='k')
943 a.axvline(thrLower, c='k')
944 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
945 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
946 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
947 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
949 # set axis limits and scales
950 a.set_ylim([1., 1.7*np.max(y)])
951 lPlot, rPlot = a.get_xlim()
952 a.set_xlim(np.array([lPlot, rPlot]))
953 a.set_yscale('log')
954 a.set_xlabel("ADU/s")
956 return