lsst.cp.pipe  19.0.0-2-gdbc0a5a+8
defects.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['FindDefectsTask',
24  'FindDefectsTaskConfig', ]
25 
26 import numpy as np
27 import os
28 import warnings
29 
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 import lsst.afw.image as afwImage
33 import lsst.meas.algorithms as measAlg
34 import lsst.afw.math as afwMath
35 import lsst.afw.detection as afwDetection
36 import lsst.afw.display as afwDisplay
37 from lsst.afw import cameraGeom
38 
39 from lsst.ip.isr import IsrTask
40 from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \
41  validateIsrConfig
42 
43 
44 class FindDefectsTaskConfig(pexConfig.Config):
45  """Config class for defect finding"""
46 
47  isrForFlats = pexConfig.ConfigurableField(
48  target=IsrTask,
49  doc="Task to perform instrumental signature removal",
50  )
51  isrForDarks = pexConfig.ConfigurableField(
52  target=IsrTask,
53  doc="Task to perform instrumental signature removal",
54  )
55  isrMandatoryStepsFlats = pexConfig.ListField(
56  dtype=str,
57  doc=("isr operations that must be performed for valid results when using flats."
58  " Raises if any of these are False"),
59  default=['doAssembleCcd', 'doFringe']
60  )
61  isrMandatoryStepsDarks = pexConfig.ListField(
62  dtype=str,
63  doc=("isr operations that must be performed for valid results when using darks. "
64  "Raises if any of these are False"),
65  default=['doAssembleCcd', 'doFringe']
66  )
67  isrForbiddenStepsFlats = pexConfig.ListField(
68  dtype=str,
69  doc=("isr operations that must NOT be performed for valid results when using flats."
70  " Raises if any of these are True"),
71  default=['doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
72  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
73  )
74  isrForbiddenStepsDarks = pexConfig.ListField(
75  dtype=str,
76  doc=("isr operations that must NOT be performed for valid results when using darks."
77  " Raises if any of these are True"),
78  default=['doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
79  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
80  )
81  isrDesirableSteps = pexConfig.ListField(
82  dtype=str,
83  doc=("isr operations that it is advisable to perform, but are not mission-critical."
84  " WARNs are logged for any of these found to be False."),
85  default=['doBias']
86  )
87  ccdKey = pexConfig.Field(
88  dtype=str,
89  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
90  default='ccd',
91  )
92  imageTypeKey = pexConfig.Field(
93  dtype=str,
94  doc="The key for the butler to use by which to check whether images are darks or flats",
95  default='imageType',
96  )
97  mode = pexConfig.ChoiceField(
98  doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?"
99  " If MASTER, a single visit number should be supplied, for which the corresponding master flat"
100  " and dark will be used. If VISITS, the list of visits will be used, treating the flats and "
101  " darks as appropriate, depending on their image types, as determined by their imageType from"
102  " config.imageTypeKey"),
103  dtype=str,
104  default="VISITS",
105  allowed={
106  "VISITS": "Calculate defects from a list of raw visits",
107  "MASTER": "Use the corresponding master calibs from the specified visit to measure defects",
108  }
109  )
110  nSigmaBright = pexConfig.Field(
111  dtype=float,
112  doc=("Number of sigma above mean for bright pixel detection. The default value was found to be",
113  " appropriate for some LSST sensors in DM-17490."),
114  default=4.8,
115  )
116  nSigmaDark = pexConfig.Field(
117  dtype=float,
118  doc=("Number of sigma below mean for dark pixel detection. The default value was found to be",
119  " appropriate for some LSST sensors in DM-17490."),
120  default=5.0,
121  )
122  nPixBorderUpDown = pexConfig.Field(
123  dtype=int,
124  doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
125  default=7,
126  )
127  nPixBorderLeftRight = pexConfig.Field(
128  dtype=int,
129  doc="Number of pixels to exclude from left & right of image when looking for defects.",
130  default=7,
131  )
132  edgesAsDefects = pexConfig.Field(
133  dtype=bool,
134  doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
135  " Normal treatment is to simply exclude this region from the defect finding, such that no"
136  " defect will be located there."),
137  default=False,
138  )
139  assertSameRun = pexConfig.Field(
140  dtype=bool,
141  doc=("Ensure that all visits are from the same run? Raises if this is not the case, or"
142  "if the run key isn't found."),
143  default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
144  )
145  combinationMode = pexConfig.ChoiceField(
146  doc="Which types of defects to identify",
147  dtype=str,
148  default="FRACTION",
149  allowed={
150  "AND": "Logical AND the pixels found in each visit to form set",
151  "OR": "Logical OR the pixels found in each visit to form set",
152  "FRACTION": "Use pixels found in more than config.combinationFraction of visits",
153  }
154  )
155  combinationFraction = pexConfig.RangeField(
156  dtype=float,
157  doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
158  " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
159  " mode AND to require pixel to appear in all images."),
160  default=0.7,
161  min=0,
162  max=1,
163  )
164  makePlots = pexConfig.Field(
165  dtype=bool,
166  doc=("Plot histograms for each visit for each amp (one plot per detector) and the final"
167  " defects overlaid on the sensor."),
168  default=False,
169  )
170  writeAs = pexConfig.ChoiceField(
171  doc="Write the output file as ASCII or FITS table",
172  dtype=str,
173  default="FITS",
174  allowed={
175  "ASCII": "Write the output as an ASCII file",
176  "FITS": "Write the output as an FITS table",
177  "BOTH": "Write the output as both a FITS table and an ASCII file",
178  }
179  )
180 
181 
182 class FindDefectsTask(pipeBase.CmdLineTask):
183  """Task for finding defects in sensors.
184 
185  The task has two modes of operation, defect finding in raws and in
186  master calibrations, which work as follows.
187 
188  Master calib defect finding
189  ----------------------------
190 
191  A single visit number is supplied, for which the corresponding flat & dark
192  will be used. This is because, at present at least, there is no way to pass
193  a calibration exposure ID from the command line to a command line task.
194 
195  The task retrieves the corresponding dark and flat exposures for the
196  supplied visit. If a flat is available the task will (be able to) look
197  for both bright and dark defects. If only a dark is found then only bright
198  defects will be sought.
199 
200  All pixels above/below the specified nSigma which lie with the specified
201  borders for flats/darks are identified as defects.
202 
203  Raw visit defect finding
204  ------------------------
205 
206  A list of exposure IDs are supplied for defect finding. The task will
207  detect bright pixels in the dark frames, if supplied, and bright & dark
208  pixels in the flats, if supplied, i.e. if you only supply darks you will
209  only be given bright defects. This is done automatically from the imageType
210  of the exposure, so the input exposure list can be a mix.
211 
212  As with the master calib detection, all pixels above/below the specified
213  nSigma which lie with the specified borders for flats/darks are identified
214  as defects. Then, a post-processing step is done to merge these detections,
215  with pixels appearing in a fraction [0..1] of the images are kept as defects
216  and those appearing below that occurrence-threshold are discarded.
217  """
218 
219  RunnerClass = SingleVisitListTaskRunner
220  ConfigClass = FindDefectsTaskConfig
221  _DefaultName = "findDefects"
222 
223  def __init__(self, *args, **kwargs):
224  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
225  self.makeSubtask("isrForFlats")
226  self.makeSubtask("isrForDarks")
227 
228  validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats,
229  self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps)
230  validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks,
231  self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps)
232  self.config.validate()
233  self.config.freeze()
234 
235  @classmethod
236  def _makeArgumentParser(cls):
237  """Augment argument parser for the FindDefectsTask."""
238  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
239  parser.add_argument("--visitList", dest="visitList", nargs="*",
240  help=("List of visits to use. Same for each detector."
241  " Uses the normal 0..10:3^234 syntax"))
242  parser.add_id_argument("--id", datasetType="newDefects",
243  ContainerClass=NonexistentDatasetTaskDataIdContainer,
244  help="The ccds to use, e.g. --id ccd=0..100")
245  return parser
246 
247  @pipeBase.timeMethod
248  def runDataRef(self, dataRef, visitList):
249  """Run the defect finding task.
250 
251  Find the defects, as described in the main task docstring, from a
252  dataRef and a list of visit(s).
253 
254  Parameters
255  ----------
256  dataRef : `lsst.daf.persistence.ButlerDataRef`
257  dataRef for the detector for the visits to be fit.
258  visitList : `list` [`int`]
259  List of visits to be processed. If config.mode == 'VISITS' then the
260  list of visits is used. If config.mode == 'MASTER' then the length
261  of visitList must be one, and the corresponding master calibrations
262  are used.
263 
264  Returns
265  -------
266  result : `lsst.pipe.base.Struct`
267  Result struct with Components:
268 
269  - ``defects`` : `lsst.meas.algorithms.Defect`
270  The defects found by the task.
271  - ``exitStatus`` : `int`
272  The exit code.
273  """
274 
275  detNum = dataRef.dataId[self.config.ccdKey]
276  self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum))
277 
278  defectLists = {'dark': [], 'flat': []}
279 
280  if self.config.mode == 'MASTER':
281  if len(visitList) > 1:
282  raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}")
283  dataRef.dataId['visit'] = visitList[0]
284 
285  for datasetType in defectLists.keys():
286  exp = dataRef.get(datasetType)
287  defects = self.findHotAndColdPixels(exp, datasetType)
288 
289  msg = "Found %s defects containing %s pixels in master %s"
290  self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
291  defectLists[datasetType].append(defects)
292  if self.config.makePlots:
293  self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType),
294  defects, datasetType)
295 
296  elif self.config.mode == 'VISITS':
297  butler = dataRef.getButler()
298 
299  if self.config.assertSameRun:
300  runs = self._getRunListFromVisits(butler, visitList)
301  if len(runs) != 1:
302  raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True")
303 
304  for visit in visitList:
305  imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'visit': visit})[0]
306  imageType = imageType.lower()
307  dataRef.dataId['visit'] = visit
308 
309  if imageType == 'flat': # note different isr tasks
310  exp = self.isrForFlats.runDataRef(dataRef).exposure
311  defects = self.findHotAndColdPixels(exp, imageType)
312  defectLists['flat'].append(defects)
313 
314  elif imageType == 'dark':
315  exp = self.isrForDarks.runDataRef(dataRef).exposure
316  defects = self.findHotAndColdPixels(exp, imageType)
317  defectLists['dark'].append(defects)
318 
319  else:
320  raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported")
321 
322  msg = "Found %s defects containing %s pixels in visit %s"
323  self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit)
324 
325  if self.config.makePlots:
326  self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType)
327 
328  msg = "Combining %s defect sets from darks for detector %s"
329  self.log.info(msg, len(defectLists['dark']), detNum)
330  mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(),
331  self.config.combinationMode)
332  msg = "Combining %s defect sets from flats for detector %s"
333  self.log.info(msg, len(defectLists['flat']), detNum)
334  mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(),
335  self.config.combinationMode)
336 
337  msg = "Combining bright and dark defect sets for detector %s"
338  self.log.info(msg, detNum)
339  brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats]
340  allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR')
341 
342  self._writeData(dataRef, allDefects)
343 
344  self.log.info("Finished finding defects in detector %s" % detNum)
345  return pipeBase.Struct(defects=allDefects, exitStatus=0)
346 
347  def _getNsigmaForPlot(self, imageType):
348  assert imageType in ['flat', 'dark']
349  nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark
350  return nSig
351 
352  @staticmethod
353  def _nPixFromDefects(defect):
354  """Count the number of pixels in a defect object."""
355  nPix = 0
356  for d in defect:
357  nPix += d.getBBox().getArea()
358  return nPix
359 
360  def _writeData(self, dataRef, defects):
361  """Write the data out to the defect file.
362 
363  Parameters
364  ----------
365  dataRef : `lsst.daf.persistence.ButlerDataRef`
366  dataRef for the detector for defects to be written.
367  defects : `lsst.meas.algorithms.Defect`
368  The defects to be written.
369  """
370  filename = dataRef.getUri(write=True) # does not guarantee that full path exists
371  dirname = os.path.dirname(filename)
372  if not os.path.exists(dirname):
373  os.makedirs(dirname)
374 
375  msg = "Writing defects to %s in format: %s"
376  self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs)
377 
378  if self.config.writeAs in ['FITS', 'BOTH']:
379  defects.writeFits(filename)
380  if self.config.writeAs in ['ASCII', 'BOTH']:
381  wroteTo = defects.writeText(filename)
382  assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0])
383  return
384 
385  @staticmethod
386  def _getRunListFromVisits(butler, visitList):
387  """Return the set of runs for the visits in visitList."""
388  runs = set()
389  for visit in visitList:
390  runs.add(butler.queryMetadata('raw', 'run', dataId={'visit': visit})[0])
391  return runs
392 
393  def _postProcessDefectSets(self, defectList, imageDimensions, mode):
394  """Combine a list of defects to make a single defect object.
395 
396  AND, OR or use percentage of visits in which defects appear
397  depending on config.
398 
399  Parameters
400  ----------
401  defectList : `list` [`lsst.meas.algorithms.Defect`]
402  The lList of defects to merge.
403  imageDimensions : `tuple` [`int`]
404  The size of the image.
405  mode : `str`
406  The combination mode to use, either 'AND', 'OR' or 'FRACTION'
407 
408  Returns
409  -------
410  defects : `lsst.meas.algorithms.Defect`
411  The defect set resulting from the merge.
412  """
413  # so that empty lists can be passed in for input data
414  # where only flats or darks are supplied
415  if defectList == []:
416  return []
417 
418  if len(defectList) == 1: # single input - no merging to do
419  return defectList[0]
420 
421  sumImage = afwImage.MaskedImageF(imageDimensions)
422  for defects in defectList:
423  for defect in defects:
424  sumImage.image[defect.getBBox()] += 1
425  sumImage /= len(defectList)
426 
427  nDetected = len(np.where(sumImage.image.array > 0)[0])
428  self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected)
429 
430  if mode == 'OR': # must appear in any
431  indices = np.where(sumImage.image.array > 0)
432  else:
433  if mode == 'AND': # must appear in all
434  threshold = 1
435  elif mode == 'FRACTION':
436  threshold = self.config.combinationFraction
437  else:
438  raise RuntimeError(f"Got unsupported combinationMode {mode}")
439  indices = np.where(sumImage.image.array >= threshold)
440 
441  BADBIT = sumImage.mask.getPlaneBitMask('BAD')
442  sumImage.mask.array[indices] |= BADBIT
443 
444  self.log.info("Post-merge %s pixels marked as defects" % len(indices[0]))
445 
446  if self.config.edgesAsDefects:
447  self.log.info("Masking edge pixels as defects in addition to previously identified defects")
448  self._setEdgeBits(sumImage, 'BAD')
449 
450  defects = measAlg.Defects.fromMask(sumImage, 'BAD')
451  return defects
452 
453  @staticmethod
454  def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
455  """Return the number of non-bad pixels in the image."""
456  nPixels = maskedIm.mask.array.size
457  nBad = countMaskedPixels(maskedIm, badMaskString)
458  return nPixels - nBad
459 
460  def findHotAndColdPixels(self, exp, imageType, setMask=False):
461  """Find hot and cold pixels in an image.
462 
463  Using config-defined thresholds on a per-amp basis, mask pixels
464  that are nSigma above threshold in dark frames (hot pixels),
465  or nSigma away from the clipped mean in flats (hot & cold pixels).
466 
467  Parameters
468  ----------
469  exp : `lsst.afw.image.exposure.Exposure`
470  The exposure in which to find defects.
471  imageType : `str`
472  The image type, either 'dark' or 'flat'.
473  setMask : `bool`
474  If true, update exp with hot and cold pixels.
475  hot: DETECTED
476  cold: DETECTED_NEGATIVE
477 
478  Returns
479  -------
480  defects : `lsst.meas.algorithms.Defect`
481  The defects found in the image.
482  """
483  assert imageType in ['flat', 'dark']
484 
485  self._setEdgeBits(exp)
486  maskedIm = exp.maskedImage
487 
488  # the detection polarity for afwDetection, True for positive,
489  # False for negative, and therefore True for darks as they only have
490  # bright pixels, and both for flats, as they have bright and dark pix
491  polarities = {'dark': [True], 'flat': [True, False]}[imageType]
492 
493  footprintList = []
494 
495  for amp in exp.getDetector():
496  ampImg = maskedIm[amp.getBBox()].clone()
497 
498  # crop ampImage depending on where the amp lies in the image
499  if self.config.nPixBorderLeftRight:
500  if ampImg.getX0() == 0:
501  ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
502  else:
503  ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
504  if self.config.nPixBorderUpDown:
505  if ampImg.getY0() == 0:
506  ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
507  else:
508  ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
509 
510  if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
511  continue
512 
513  ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
514 
515  mergedSet = None
516  for polarity in polarities:
517  nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark
518  threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
519 
520  footprintSet = afwDetection.FootprintSet(ampImg, threshold)
521  if setMask:
522  footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
523 
524  if mergedSet is None:
525  mergedSet = footprintSet
526  else:
527  mergedSet.merge(footprintSet)
528 
529  footprintList += mergedSet.getFootprints()
530 
531  defects = measAlg.Defects.fromFootprintList(footprintList)
532  return defects
533 
534  def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
535  """Set edge bits on an exposure or maskedImage.
536 
537  Raises
538  ------
539  TypeError
540  Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
541  """
542  if isinstance(exposureOrMaskedImage, afwImage.Exposure):
543  mi = exposureOrMaskedImage.maskedImage
544  elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
545  mi = exposureOrMaskedImage
546  else:
547  t = type(exposureOrMaskedImage)
548  raise TypeError(f"Function supports exposure or maskedImage but not {t}")
549 
550  MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
551  if self.config.nPixBorderLeftRight:
552  mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
553  mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
554  if self.config.nPixBorderUpDown:
555  mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
556  mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
557 
558  def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover
559  """Plot the defects and pixel histograms.
560 
561  Parameters
562  ----------
563  dataRef : `lsst.daf.persistence.ButlerDataRef`
564  dataRef for the detector.
565  exp : `lsst.afw.image.exposure.Exposure`
566  The exposure in which the defects were found.
567  visit : `int`
568  The visit number.
569  nSig : `float`
570  The number of sigma used for detection
571  defects : `lsst.meas.algorithms.Defect`
572  The defects to plot.
573  imageType : `str`
574  The type of image, either 'dark' or 'flat'.
575 
576  Currently only for LSST sensors. Plots are written to the path
577  given by the butler for the ``cpPipePlotRoot`` dataset type.
578  """
579  import matplotlib.pyplot as plt
580  from matplotlib.backends.backend_pdf import PdfPages
581 
582  afwDisplay.setDefaultBackend("matplotlib")
583  plt.interactive(False) # seems to need reasserting here
584 
585  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
586  if not os.path.exists(dirname):
587  os.makedirs(dirname)
588 
589  detNum = exp.getDetector().getId()
590  nAmps = len(exp.getDetector())
591 
592  if self.config.mode == "MASTER":
593  filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf"
594  elif self.config.mode == "VISITS":
595  filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf"
596 
597  filenameFull = os.path.join(dirname, filename)
598 
599  with warnings.catch_warnings():
600  msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure."
601  warnings.filterwarnings("ignore", message=msg)
602  with PdfPages(filenameFull) as pdfPages:
603  if nAmps == 16:
604  self._plotAmpHistogram(dataRef, exp, visit, nSig)
605  pdfPages.savefig()
606 
607  self._plotDefects(exp, visit, defects, imageType)
608  pdfPages.savefig()
609  self.log.info("Wrote plot(s) to %s" % filenameFull)
610 
611  def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover
612  """Plot the defects found by the task.
613 
614  Parameters
615  ----------
616  exp : `lsst.afw.image.exposure.Exposure`
617  The exposure in which the defects were found.
618  visit : `int`
619  The visit number.
620  defects : `lsst.meas.algorithms.Defect`
621  The defects to plot.
622  imageType : `str`
623  The type of image, either 'dark' or 'flat'.
624  """
625  expCopy = exp.clone() # we mess with the copy later, so make a clone
626  del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes
627  maskedIm = expCopy.maskedImage
628 
629  defects.maskPixels(expCopy.maskedImage, "BAD")
630  detector = expCopy.getDetector()
631 
632  disp = afwDisplay.Display(0, reopenPlot=True, dpi=200)
633 
634  if imageType == "flat": # set each amp image to have a mean of 1.00
635  for amp in detector:
636  ampIm = maskedIm.image[amp.getBBox()]
637  ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1
638 
639  mpDict = maskedIm.mask.getMaskPlaneDict()
640  for plane in mpDict.keys():
641  if plane in ['BAD']:
642  continue
643  disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
644 
645  disp.scale('asinh', 'zscale')
646  disp.setMaskTransparency(80)
647  disp.setMaskPlaneColor("BAD", afwDisplay.RED)
648 
649  disp.setImageColormap('gray')
650  title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}"
651  f", Type: {imageType}, visit: {visit}")
652  disp.mtv(maskedIm, title=title)
653 
654  cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
655 
656  def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover
657  """
658  Make a histogram of the distribution of pixel values for each amp.
659 
660  The main image data histogram is plotted in blue. Edge pixels,
661  if masked, are in red. Note that masked edge pixels do not contribute
662  to the underflow and overflow numbers.
663 
664  Note that this currently only supports the 16-amp LSST detectors.
665 
666  Parameters
667  ----------
668  dataRef : `lsst.daf.persistence.ButlerDataRef`
669  dataRef for the detector.
670  exp : `lsst.afw.image.exposure.Exposure`
671  The exposure in which the defects were found.
672  visit : `int`
673  The visit number.
674  nSigmaUsed : `float`
675  The number of sigma used for detection
676  """
677  import matplotlib.pyplot as plt
678 
679  detector = exp.getDetector()
680 
681  if len(detector) != 16:
682  raise RuntimeError("Plotting currently only supported for 16 amp detectors")
683  fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
684 
685  expTime = exp.getInfo().getVisitInfo().getExposureTime()
686 
687  for (amp, a) in zip(reversed(detector), ax.flatten()):
688  mi = exp.maskedImage[amp.getBBox()]
689 
690  # normalize by expTime as we plot in ADU/s and don't always work with master calibs
691  mi.image.array /= expTime
692  stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
693  mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
694 
695  # Get array of pixels
696  EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
697  imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
698  edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
699 
700  thrUpper = mean + nSigmaUsed*sigma
701  thrLower = mean - nSigmaUsed*sigma
702 
703  nRight = len(imgData[imgData > thrUpper])
704  nLeft = len(imgData[imgData < thrLower])
705 
706  nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
707  leftEdge = mean - nsig * nSigmaUsed*sigma
708  rightEdge = mean + nsig * nSigmaUsed*sigma
709  nbins = np.linspace(leftEdge, rightEdge, 1000)
710  ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red')
711  y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue')
712 
713  # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram
714  nOverflow = len(imgData[imgData > rightEdge])
715  nUnderflow = len(imgData[imgData < leftEdge])
716 
717  # Put v-lines and textboxes in
718  a.axvline(thrUpper, c='k')
719  a.axvline(thrLower, c='k')
720  msg = f"{amp.getName()}\nmean:{mean: .1f}\n$\\sigma$:{sigma: .1f}"
721  a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
722  msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
723  a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
724 
725  # set axis limits and scales
726  a.set_ylim([1., 1.7*np.max(y)])
727  lPlot, rPlot = a.get_xlim()
728  a.set_xlim(np.array([lPlot, rPlot]))
729  a.set_yscale('log')
730  a.set_xlabel("ADU/s")
731 
732  return
def _getNsigmaForPlot(self, imageType)
Definition: defects.py:347
def runDataRef(self, dataRef, visitList)
Definition: defects.py:248
def _getRunListFromVisits(butler, visitList)
Definition: defects.py:386
def countMaskedPixels(maskedIm, maskPlane)
Definition: utils.py:35
def _plotDefects(self, exp, visit, defects, imageType)
Definition: defects.py:611
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:200
def findHotAndColdPixels(self, exp, imageType, setMask=False)
Definition: defects.py:460
def _postProcessDefectSets(self, defectList, imageDimensions, mode)
Definition: defects.py:393
def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE')
Definition: defects.py:534
def _writeData(self, dataRef, defects)
Definition: defects.py:360
def __init__(self, args, kwargs)
Definition: defects.py:223
def _plot(self, dataRef, exp, visit, nSig, defects, imageType)
Definition: defects.py:558
def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA")
Definition: defects.py:454
def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed)
Definition: defects.py:656