lsst.cp.pipe  19.0.0-6-gcd25531
defects.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['FindDefectsTask',
24  'FindDefectsTaskConfig', ]
25 
26 import numpy as np
27 import os
28 import warnings
29 
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 import lsst.afw.image as afwImage
33 import lsst.meas.algorithms as measAlg
34 import lsst.afw.math as afwMath
35 import lsst.afw.detection as afwDetection
36 import lsst.afw.display as afwDisplay
37 from lsst.afw import cameraGeom
38 from lsst.geom import Box2I, Point2I
39 
40 from lsst.ip.isr import IsrTask
41 from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \
42  validateIsrConfig
43 
44 
45 class FindDefectsTaskConfig(pexConfig.Config):
46  """Config class for defect finding"""
47 
48  isrForFlats = pexConfig.ConfigurableField(
49  target=IsrTask,
50  doc="Task to perform instrumental signature removal",
51  )
52  isrForDarks = pexConfig.ConfigurableField(
53  target=IsrTask,
54  doc="Task to perform instrumental signature removal",
55  )
56  isrMandatoryStepsFlats = pexConfig.ListField(
57  dtype=str,
58  doc=("isr operations that must be performed for valid results when using flats."
59  " Raises if any of these are False"),
60  default=['doAssembleCcd', 'doFringe']
61  )
62  isrMandatoryStepsDarks = pexConfig.ListField(
63  dtype=str,
64  doc=("isr operations that must be performed for valid results when using darks. "
65  "Raises if any of these are False"),
66  default=['doAssembleCcd', 'doFringe']
67  )
68  isrForbiddenStepsFlats = pexConfig.ListField(
69  dtype=str,
70  doc=("isr operations that must NOT be performed for valid results when using flats."
71  " Raises if any of these are True"),
72  default=['doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
73  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
74  )
75  isrForbiddenStepsDarks = pexConfig.ListField(
76  dtype=str,
77  doc=("isr operations that must NOT be performed for valid results when using darks."
78  " Raises if any of these are True"),
79  default=['doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
80  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
81  )
82  isrDesirableSteps = pexConfig.ListField(
83  dtype=str,
84  doc=("isr operations that it is advisable to perform, but are not mission-critical."
85  " WARNs are logged for any of these found to be False."),
86  default=['doBias']
87  )
88  ccdKey = pexConfig.Field(
89  dtype=str,
90  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
91  default='ccd',
92  )
93  imageTypeKey = pexConfig.Field(
94  dtype=str,
95  doc="The key for the butler to use by which to check whether images are darks or flats",
96  default='imageType',
97  )
98  mode = pexConfig.ChoiceField(
99  doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?"
100  " If MASTER, a single visit number should be supplied, for which the corresponding master flat"
101  " and dark will be used. If VISITS, the list of visits will be used, treating the flats and "
102  " darks as appropriate, depending on their image types, as determined by their imageType from"
103  " config.imageTypeKey"),
104  dtype=str,
105  default="VISITS",
106  allowed={
107  "VISITS": "Calculate defects from a list of raw visits",
108  "MASTER": "Use the corresponding master calibs from the specified visit to measure defects",
109  }
110  )
111  nSigmaBright = pexConfig.Field(
112  dtype=float,
113  doc=("Number of sigma above mean for bright pixel detection. The default value was found to be",
114  " appropriate for some LSST sensors in DM-17490."),
115  default=4.8,
116  )
117  nSigmaDark = pexConfig.Field(
118  dtype=float,
119  doc=("Number of sigma below mean for dark pixel detection. The default value was found to be",
120  " appropriate for some LSST sensors in DM-17490."),
121  default=5.0,
122  )
123  nPixBorderUpDown = pexConfig.Field(
124  dtype=int,
125  doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
126  default=7,
127  )
128  nPixBorderLeftRight = pexConfig.Field(
129  dtype=int,
130  doc="Number of pixels to exclude from left & right of image when looking for defects.",
131  default=7,
132  )
133  badOnAndOffPixelColumnThreshold = pexConfig.Field(
134  dtype=int,
135  doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ",
136  "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ",
137  "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ",
138  "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ",
139  "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ",
140  "non-bad pixels in BPC, an exception to the above is made and those consecutive ",
141  "'goodPixelColumnGapThreshold' are not marked as bad."),
142  default=50,
143  )
144  goodPixelColumnGapThreshold = pexConfig.Field(
145  dtype=int,
146  doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ",
147  "'badOnAndOffPixelColumnThreshold')."),
148  default=30,
149  )
150  edgesAsDefects = pexConfig.Field(
151  dtype=bool,
152  doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
153  " Normal treatment is to simply exclude this region from the defect finding, such that no"
154  " defect will be located there."),
155  default=False,
156  )
157  assertSameRun = pexConfig.Field(
158  dtype=bool,
159  doc=("Ensure that all visits are from the same run? Raises if this is not the case, or"
160  "if the run key isn't found."),
161  default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
162  )
163  combinationMode = pexConfig.ChoiceField(
164  doc="Which types of defects to identify",
165  dtype=str,
166  default="FRACTION",
167  allowed={
168  "AND": "Logical AND the pixels found in each visit to form set ",
169  "OR": "Logical OR the pixels found in each visit to form set ",
170  "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
171  }
172  )
173  combinationFraction = pexConfig.RangeField(
174  dtype=float,
175  doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
176  " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
177  " mode AND to require pixel to appear in all images."),
178  default=0.7,
179  min=0,
180  max=1,
181  )
182  makePlots = pexConfig.Field(
183  dtype=bool,
184  doc=("Plot histograms for each visit for each amp (one plot per detector) and the final"
185  " defects overlaid on the sensor."),
186  default=False,
187  )
188  writeAs = pexConfig.ChoiceField(
189  doc="Write the output file as ASCII or FITS table",
190  dtype=str,
191  default="FITS",
192  allowed={
193  "ASCII": "Write the output as an ASCII file",
194  "FITS": "Write the output as an FITS table",
195  "BOTH": "Write the output as both a FITS table and an ASCII file",
196  }
197  )
198 
199 
200 class FindDefectsTask(pipeBase.CmdLineTask):
201  """Task for finding defects in sensors.
202 
203  The task has two modes of operation, defect finding in raws and in
204  master calibrations, which work as follows.
205 
206  Master calib defect finding
207  ----------------------------
208 
209  A single visit number is supplied, for which the corresponding flat & dark
210  will be used. This is because, at present at least, there is no way to pass
211  a calibration exposure ID from the command line to a command line task.
212 
213  The task retrieves the corresponding dark and flat exposures for the
214  supplied visit. If a flat is available the task will (be able to) look
215  for both bright and dark defects. If only a dark is found then only bright
216  defects will be sought.
217 
218  All pixels above/below the specified nSigma which lie with the specified
219  borders for flats/darks are identified as defects.
220 
221  Raw visit defect finding
222  ------------------------
223 
224  A list of exposure IDs are supplied for defect finding. The task will
225  detect bright pixels in the dark frames, if supplied, and bright & dark
226  pixels in the flats, if supplied, i.e. if you only supply darks you will
227  only be given bright defects. This is done automatically from the imageType
228  of the exposure, so the input exposure list can be a mix.
229 
230  As with the master calib detection, all pixels above/below the specified
231  nSigma which lie with the specified borders for flats/darks are identified
232  as defects. Then, a post-processing step is done to merge these detections,
233  with pixels appearing in a fraction [0..1] of the images are kept as defects
234  and those appearing below that occurrence-threshold are discarded.
235  """
236 
237  RunnerClass = SingleVisitListTaskRunner
238  ConfigClass = FindDefectsTaskConfig
239  _DefaultName = "findDefects"
240 
241  def __init__(self, *args, **kwargs):
242  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
243  self.makeSubtask("isrForFlats")
244  self.makeSubtask("isrForDarks")
245 
246  validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats,
247  self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps)
248  validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks,
249  self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps)
250  self.config.validate()
251  self.config.freeze()
252 
253  @classmethod
254  def _makeArgumentParser(cls):
255  """Augment argument parser for the FindDefectsTask."""
256  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
257  parser.add_argument("--visitList", dest="visitList", nargs="*",
258  help=("List of visits to use. Same for each detector."
259  " Uses the normal 0..10:3^234 syntax"))
260  parser.add_id_argument("--id", datasetType="newDefects",
261  ContainerClass=NonexistentDatasetTaskDataIdContainer,
262  help="The ccds to use, e.g. --id ccd=0..100")
263  return parser
264 
265  @pipeBase.timeMethod
266  def runDataRef(self, dataRef, visitList):
267  """Run the defect finding task.
268 
269  Find the defects, as described in the main task docstring, from a
270  dataRef and a list of visit(s).
271 
272  Parameters
273  ----------
274  dataRef : `lsst.daf.persistence.ButlerDataRef`
275  dataRef for the detector for the visits to be fit.
276  visitList : `list` [`int`]
277  List of visits to be processed. If config.mode == 'VISITS' then the
278  list of visits is used. If config.mode == 'MASTER' then the length
279  of visitList must be one, and the corresponding master calibrations
280  are used.
281 
282  Returns
283  -------
284  result : `lsst.pipe.base.Struct`
285  Result struct with Components:
286 
287  - ``defects`` : `lsst.meas.algorithms.Defect`
288  The defects found by the task.
289  - ``exitStatus`` : `int`
290  The exit code.
291  """
292 
293  detNum = dataRef.dataId[self.config.ccdKey]
294  self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum))
295 
296  defectLists = {'dark': [], 'flat': []}
297 
298  if self.config.mode == 'MASTER':
299  if len(visitList) > 1:
300  raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}")
301  dataRef.dataId['visit'] = visitList[0]
302 
303  for datasetType in defectLists.keys():
304  exp = dataRef.get(datasetType)
305  defects = self.findHotAndColdPixels(exp, datasetType)
306 
307  msg = "Found %s defects containing %s pixels in master %s"
308  self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
309  defectLists[datasetType].append(defects)
310  if self.config.makePlots:
311  self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType),
312  defects, datasetType)
313 
314  elif self.config.mode == 'VISITS':
315  butler = dataRef.getButler()
316 
317  if self.config.assertSameRun:
318  runs = self._getRunListFromVisits(butler, visitList)
319  if len(runs) != 1:
320  raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True")
321 
322  for visit in visitList:
323  imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'visit': visit})[0]
324  imageType = imageType.lower()
325  dataRef.dataId['visit'] = visit
326  if imageType == 'flat': # note different isr tasks
327  exp = self.isrForFlats.runDataRef(dataRef).exposure
328  defects = self.findHotAndColdPixels(exp, imageType)
329  defectLists['flat'].append(defects)
330 
331  elif imageType == 'dark':
332  exp = self.isrForDarks.runDataRef(dataRef).exposure
333  defects = self.findHotAndColdPixels(exp, imageType)
334  defectLists['dark'].append(defects)
335 
336  else:
337  raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported")
338 
339  msg = "Found %s defects containing %s pixels in visit %s"
340  self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit)
341 
342  if self.config.makePlots:
343  self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType)
344 
345  msg = "Combining %s defect sets from darks for detector %s"
346  self.log.info(msg, len(defectLists['dark']), detNum)
347  mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(),
348  self.config.combinationMode)
349  msg = "Combining %s defect sets from flats for detector %s"
350  self.log.info(msg, len(defectLists['flat']), detNum)
351  mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(),
352  self.config.combinationMode)
353 
354  msg = "Combining bright and dark defect sets for detector %s"
355  self.log.info(msg, detNum)
356  brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats]
357  allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR')
358 
359  self._writeData(dataRef, allDefects)
360 
361  self.log.info("Finished finding defects in detector %s" % detNum)
362  return pipeBase.Struct(defects=allDefects, exitStatus=0)
363 
364  def _getNsigmaForPlot(self, imageType):
365  assert imageType in ['flat', 'dark']
366  nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark
367  return nSig
368 
369  @staticmethod
370  def _nPixFromDefects(defect):
371  """Count the number of pixels in a defect object."""
372  nPix = 0
373  for d in defect:
374  nPix += d.getBBox().getArea()
375  return nPix
376 
377  def _writeData(self, dataRef, defects):
378  """Write the data out to the defect file.
379 
380  Parameters
381  ----------
382  dataRef : `lsst.daf.persistence.ButlerDataRef`
383  dataRef for the detector for defects to be written.
384  defects : `lsst.meas.algorithms.Defect`
385  The defects to be written.
386  """
387  filename = dataRef.getUri(write=True) # does not guarantee that full path exists
388  dirname = os.path.dirname(filename)
389  if not os.path.exists(dirname):
390  os.makedirs(dirname)
391 
392  msg = "Writing defects to %s in format: %s"
393  self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs)
394 
395  if self.config.writeAs in ['FITS', 'BOTH']:
396  defects.writeFits(filename)
397  if self.config.writeAs in ['ASCII', 'BOTH']:
398  wroteTo = defects.writeText(filename)
399  assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0])
400  return
401 
402  @staticmethod
403  def _getRunListFromVisits(butler, visitList):
404  """Return the set of runs for the visits in visitList."""
405  runs = set()
406  for visit in visitList:
407  runs.add(butler.queryMetadata('raw', 'run', dataId={'visit': visit})[0])
408  return runs
409 
410  def _postProcessDefectSets(self, defectList, imageDimensions, mode):
411  """Combine a list of defects to make a single defect object.
412 
413  AND, OR or use percentage of visits in which defects appear
414  depending on config.
415 
416  Parameters
417  ----------
418  defectList : `list` [`lsst.meas.algorithms.Defect`]
419  The lList of defects to merge.
420  imageDimensions : `tuple` [`int`]
421  The size of the image.
422  mode : `str`
423  The combination mode to use, either 'AND', 'OR' or 'FRACTION'
424 
425  Returns
426  -------
427  defects : `lsst.meas.algorithms.Defect`
428  The defect set resulting from the merge.
429  """
430  # so that empty lists can be passed in for input data
431  # where only flats or darks are supplied
432  if defectList == []:
433  return []
434 
435  if len(defectList) == 1: # single input - no merging to do
436  return defectList[0]
437 
438  sumImage = afwImage.MaskedImageF(imageDimensions)
439  for defects in defectList:
440  for defect in defects:
441  sumImage.image[defect.getBBox()] += 1
442  sumImage /= len(defectList)
443 
444  nDetected = len(np.where(sumImage.image.array > 0)[0])
445  self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected)
446 
447  if mode == 'OR': # must appear in any
448  indices = np.where(sumImage.image.array > 0)
449  else:
450  if mode == 'AND': # must appear in all
451  threshold = 1
452  elif mode == 'FRACTION':
453  threshold = self.config.combinationFraction
454  else:
455  raise RuntimeError(f"Got unsupported combinationMode {mode}")
456  indices = np.where(sumImage.image.array >= threshold)
457 
458  BADBIT = sumImage.mask.getPlaneBitMask('BAD')
459  sumImage.mask.array[indices] |= BADBIT
460 
461  self.log.info("Post-merge %s pixels marked as defects" % len(indices[0]))
462 
463  if self.config.edgesAsDefects:
464  self.log.info("Masking edge pixels as defects in addition to previously identified defects")
465  self._setEdgeBits(sumImage, 'BAD')
466 
467  defects = measAlg.Defects.fromMask(sumImage, 'BAD')
468  return defects
469 
470  @staticmethod
471  def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
472  """Return the number of non-bad pixels in the image."""
473  nPixels = maskedIm.mask.array.size
474  nBad = countMaskedPixels(maskedIm, badMaskString)
475  return nPixels - nBad
476 
477  def findHotAndColdPixels(self, exp, imageType, setMask=False):
478  """Find hot and cold pixels in an image.
479 
480  Using config-defined thresholds on a per-amp basis, mask pixels
481  that are nSigma above threshold in dark frames (hot pixels),
482  or nSigma away from the clipped mean in flats (hot & cold pixels).
483 
484  Parameters
485  ----------
486  exp : `lsst.afw.image.exposure.Exposure`
487  The exposure in which to find defects.
488  imageType : `str`
489  The image type, either 'dark' or 'flat'.
490  setMask : `bool`
491  If true, update exp with hot and cold pixels.
492  hot: DETECTED
493  cold: DETECTED_NEGATIVE
494 
495  Returns
496  -------
497  defects : `lsst.meas.algorithms.Defect`
498  The defects found in the image.
499  """
500  assert imageType in ['flat', 'dark']
501 
502  self._setEdgeBits(exp)
503  maskedIm = exp.maskedImage
504 
505  # the detection polarity for afwDetection, True for positive,
506  # False for negative, and therefore True for darks as they only have
507  # bright pixels, and both for flats, as they have bright and dark pix
508  polarities = {'dark': [True], 'flat': [True, False]}[imageType]
509 
510  footprintList = []
511 
512  for amp in exp.getDetector():
513  ampImg = maskedIm[amp.getBBox()].clone()
514 
515  # crop ampImage depending on where the amp lies in the image
516  if self.config.nPixBorderLeftRight:
517  if ampImg.getX0() == 0:
518  ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
519  else:
520  ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
521  if self.config.nPixBorderUpDown:
522  if ampImg.getY0() == 0:
523  ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
524  else:
525  ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
526 
527  if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
528  continue
529 
530  ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
531 
532  mergedSet = None
533  for polarity in polarities:
534  nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark
535  threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
536 
537  footprintSet = afwDetection.FootprintSet(ampImg, threshold)
538  if setMask:
539  footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
540 
541  if mergedSet is None:
542  mergedSet = footprintSet
543  else:
544  mergedSet.merge(footprintSet)
545 
546  footprintList += mergedSet.getFootprints()
547 
548  defects = measAlg.Defects.fromFootprintList(footprintList)
549  defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
550 
551  return defects
552 
554  """Mask blocks in a column if there are on-and-off bad pixels
555 
556  If there's a column with on and off bad pixels, mask all the pixels in between,
557  except if there is a large enough gap of consecutive good pixels between two
558  bad pixels in the column.
559 
560  Parameters
561  ---------
562  defects: `lsst.meas.algorithms.Defect`
563  The defects found in the image so far
564 
565  Returns
566  ------
567  defects: `lsst.meas.algorithms.Defect`
568  If the number of bad pixels in a column is not larger or equal than
569  self.config.badPixelColumnThreshold, the iput list is returned. Otherwise,
570  the defects list returned will include boxes that mask blocks of on-and-of
571  pixels.
572  """
573  # Get the (x, y) values of each bad pixel in amp.
574  coordinates = []
575  for defect in defects:
576  bbox = defect.getBBox()
577  x0, y0 = bbox.getMinX(), bbox.getMinY()
578  deltaX0, deltaY0 = bbox.getDimensions()
579  for j in np.arange(y0, y0+deltaY0):
580  for i in np.arange(x0, x0 + deltaX0):
581  coordinates.append((i, j))
582 
583  x, y = [], []
584  for coordinatePair in coordinates:
585  x.append(coordinatePair[0])
586  y.append(coordinatePair[1])
587 
588  x = np.array(x)
589  y = np.array(y)
590  # Find the defects with same "x" (vertical) coordinate (column).
591  unique, counts = np.unique(x, return_counts=True)
592  multipleX = []
593  for (a, b) in zip(unique, counts):
594  if b >= self.config.badOnAndOffPixelColumnThreshold:
595  multipleX.append(a)
596  if len(multipleX) != 0:
597  defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
598 
599  return defects
600 
601  def _markBlocksInBadColumn(self, x, y, multipleX, defects):
602  """Mask blocks in a column if number of on-and-off bad pixels is above threshold.
603 
604  This function is called if the number of on-and-off bad pixels in a column
605  is larger or equal than self.config.badOnAndOffPixelColumnThreshold.
606 
607  Parameters
608  ---------
609  x: list
610  Lower left x coordinate of defect box. x coordinate is along the short axis if amp.
611 
612  y: list
613  Lower left y coordinate of defect box. x coordinate is along the long axis if amp.
614 
615  multipleX: list
616  List of x coordinates in amp. with multiple bad pixels (i.e., columns with defects).
617 
618  defects: `lsst.meas.algorithms.Defect`
619  The defcts found in the image so far
620 
621  Returns
622  -------
623  defects: `lsst.meas.algorithms.Defect`
624  The defects list returned that will include boxes that mask blocks
625  of on-and-of pixels.
626  """
627  goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
628  for x0 in multipleX:
629  index = np.where(x == x0)
630  multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
631  minY, maxY = np.min(multipleY), np.max(multipleY)
632  # Next few lines: don't mask pixels in column if gap of good pixels between
633  # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'.
634  diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
635  if len(diffIndex) != 0:
636  limits = [minY] # put the minimum first
637  for gapIndex in diffIndex:
638  limits.append(multipleY[gapIndex])
639  limits.append(multipleY[gapIndex+1])
640  limits.append(maxY) # maximum last
641  assert len(limits)%2 == 0, 'limits is even by design, but check anyways'
642  for i in np.arange(0, len(limits)-1, 2):
643  s = Box2I(minimum = Point2I(x0, limits[i]), maximum = Point2I(x0, limits[i+1]))
644  if s not in defects:
645  defects.append(s)
646  else: # No gap is large enough
647  s = Box2I(minimum = Point2I(x0, minY), maximum = Point2I(x0, maxY))
648  if s not in defects:
649  defects.append(s)
650  return defects
651 
652  def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
653  """Set edge bits on an exposure or maskedImage.
654 
655  Raises
656  ------
657  TypeError
658  Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
659  """
660  if isinstance(exposureOrMaskedImage, afwImage.Exposure):
661  mi = exposureOrMaskedImage.maskedImage
662  elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
663  mi = exposureOrMaskedImage
664  else:
665  t = type(exposureOrMaskedImage)
666  raise TypeError(f"Function supports exposure or maskedImage but not {t}")
667 
668  MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
669  if self.config.nPixBorderLeftRight:
670  mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
671  mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
672  if self.config.nPixBorderUpDown:
673  mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
674  mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
675 
676  def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover
677  """Plot the defects and pixel histograms.
678 
679  Parameters
680  ----------
681  dataRef : `lsst.daf.persistence.ButlerDataRef`
682  dataRef for the detector.
683  exp : `lsst.afw.image.exposure.Exposure`
684  The exposure in which the defects were found.
685  visit : `int`
686  The visit number.
687  nSig : `float`
688  The number of sigma used for detection
689  defects : `lsst.meas.algorithms.Defect`
690  The defects to plot.
691  imageType : `str`
692  The type of image, either 'dark' or 'flat'.
693 
694  Currently only for LSST sensors. Plots are written to the path
695  given by the butler for the ``cpPipePlotRoot`` dataset type.
696  """
697  import matplotlib.pyplot as plt
698  from matplotlib.backends.backend_pdf import PdfPages
699 
700  afwDisplay.setDefaultBackend("matplotlib")
701  plt.interactive(False) # seems to need reasserting here
702 
703  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
704  if not os.path.exists(dirname):
705  os.makedirs(dirname)
706 
707  detNum = exp.getDetector().getId()
708  nAmps = len(exp.getDetector())
709 
710  if self.config.mode == "MASTER":
711  filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf"
712  elif self.config.mode == "VISITS":
713  filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf"
714 
715  filenameFull = os.path.join(dirname, filename)
716 
717  with warnings.catch_warnings():
718  msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure."
719  warnings.filterwarnings("ignore", message=msg)
720  with PdfPages(filenameFull) as pdfPages:
721  if nAmps == 16:
722  self._plotAmpHistogram(dataRef, exp, visit, nSig)
723  pdfPages.savefig()
724 
725  self._plotDefects(exp, visit, defects, imageType)
726  pdfPages.savefig()
727  self.log.info("Wrote plot(s) to %s" % filenameFull)
728 
729  def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover
730  """Plot the defects found by the task.
731 
732  Parameters
733  ----------
734  exp : `lsst.afw.image.exposure.Exposure`
735  The exposure in which the defects were found.
736  visit : `int`
737  The visit number.
738  defects : `lsst.meas.algorithms.Defect`
739  The defects to plot.
740  imageType : `str`
741  The type of image, either 'dark' or 'flat'.
742  """
743  expCopy = exp.clone() # we mess with the copy later, so make a clone
744  del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes
745  maskedIm = expCopy.maskedImage
746 
747  defects.maskPixels(expCopy.maskedImage, "BAD")
748  detector = expCopy.getDetector()
749 
750  disp = afwDisplay.Display(0, reopenPlot=True, dpi=200)
751 
752  if imageType == "flat": # set each amp image to have a mean of 1.00
753  for amp in detector:
754  ampIm = maskedIm.image[amp.getBBox()]
755  ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1
756 
757  mpDict = maskedIm.mask.getMaskPlaneDict()
758  for plane in mpDict.keys():
759  if plane in ['BAD']:
760  continue
761  disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
762 
763  disp.scale('asinh', 'zscale')
764  disp.setMaskTransparency(80)
765  disp.setMaskPlaneColor("BAD", afwDisplay.RED)
766 
767  disp.setImageColormap('gray')
768  title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}"
769  f", Type: {imageType}, visit: {visit}")
770  disp.mtv(maskedIm, title=title)
771 
772  cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
773 
774  def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover
775  """
776  Make a histogram of the distribution of pixel values for each amp.
777 
778  The main image data histogram is plotted in blue. Edge pixels,
779  if masked, are in red. Note that masked edge pixels do not contribute
780  to the underflow and overflow numbers.
781 
782  Note that this currently only supports the 16-amp LSST detectors.
783 
784  Parameters
785  ----------
786  dataRef : `lsst.daf.persistence.ButlerDataRef`
787  dataRef for the detector.
788  exp : `lsst.afw.image.exposure.Exposure`
789  The exposure in which the defects were found.
790  visit : `int`
791  The visit number.
792  nSigmaUsed : `float`
793  The number of sigma used for detection
794  """
795  import matplotlib.pyplot as plt
796 
797  detector = exp.getDetector()
798 
799  if len(detector) != 16:
800  raise RuntimeError("Plotting currently only supported for 16 amp detectors")
801  fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
802 
803  expTime = exp.getInfo().getVisitInfo().getExposureTime()
804 
805  for (amp, a) in zip(reversed(detector), ax.flatten()):
806  mi = exp.maskedImage[amp.getBBox()]
807 
808  # normalize by expTime as we plot in ADU/s and don't always work with master calibs
809  mi.image.array /= expTime
810  stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
811  mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
812 
813  # Get array of pixels
814  EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
815  imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
816  edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
817 
818  thrUpper = mean + nSigmaUsed*sigma
819  thrLower = mean - nSigmaUsed*sigma
820 
821  nRight = len(imgData[imgData > thrUpper])
822  nLeft = len(imgData[imgData < thrLower])
823 
824  nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
825  leftEdge = mean - nsig * nSigmaUsed*sigma
826  rightEdge = mean + nsig * nSigmaUsed*sigma
827  nbins = np.linspace(leftEdge, rightEdge, 1000)
828  ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red')
829  y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue')
830 
831  # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram
832  nOverflow = len(imgData[imgData > rightEdge])
833  nUnderflow = len(imgData[imgData < leftEdge])
834 
835  # Put v-lines and textboxes in
836  a.axvline(thrUpper, c='k')
837  a.axvline(thrLower, c='k')
838  msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
839  a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
840  msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
841  a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
842 
843  # set axis limits and scales
844  a.set_ylim([1., 1.7*np.max(y)])
845  lPlot, rPlot = a.get_xlim()
846  a.set_xlim(np.array([lPlot, rPlot]))
847  a.set_yscale('log')
848  a.set_xlabel("ADU/s")
849 
850  return
def _getNsigmaForPlot(self, imageType)
Definition: defects.py:364
def runDataRef(self, dataRef, visitList)
Definition: defects.py:266
def _getRunListFromVisits(butler, visitList)
Definition: defects.py:403
def countMaskedPixels(maskedIm, maskPlane)
Definition: utils.py:35
def _plotDefects(self, exp, visit, defects, imageType)
Definition: defects.py:729
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def _markBlocksInBadColumn(self, x, y, multipleX, defects)
Definition: defects.py:601
def findHotAndColdPixels(self, exp, imageType, setMask=False)
Definition: defects.py:477
def _postProcessDefectSets(self, defectList, imageDimensions, mode)
Definition: defects.py:410
def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE')
Definition: defects.py:652
def _writeData(self, dataRef, defects)
Definition: defects.py:377
def maskBlocksIfIntermitentBadPixelsInColumn(self, defects)
Definition: defects.py:553
def __init__(self, args, kwargs)
Definition: defects.py:241
def _plot(self, dataRef, exp, visit, nSig, defects, imageType)
Definition: defects.py:676
def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA")
Definition: defects.py:471
def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed)
Definition: defects.py:774