lsst.cp.pipe  20.0.0-22-gc512666+f30f3883a7
defects.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 #
22 
23 __all__ = ['FindDefectsTask',
24  'FindDefectsTaskConfig', ]
25 
26 import numpy as np
27 import os
28 import warnings
29 
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 import lsst.afw.image as afwImage
33 import lsst.meas.algorithms as measAlg
34 import lsst.afw.math as afwMath
35 import lsst.afw.detection as afwDetection
36 import lsst.afw.display as afwDisplay
37 from lsst.afw import cameraGeom
38 from lsst.geom import Box2I, Point2I
39 import lsst.daf.base as dafBase
40 
41 from lsst.ip.isr import IsrTask
42 from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \
43  validateIsrConfig
44 
45 
46 class FindDefectsTaskConfig(pexConfig.Config):
47  """Config class for defect finding"""
48 
49  isrForFlats = pexConfig.ConfigurableField(
50  target=IsrTask,
51  doc="Task to perform instrumental signature removal",
52  )
53  isrForDarks = pexConfig.ConfigurableField(
54  target=IsrTask,
55  doc="Task to perform instrumental signature removal",
56  )
57  isrMandatoryStepsFlats = pexConfig.ListField(
58  dtype=str,
59  doc=("isr operations that must be performed for valid results when using flats."
60  " Raises if any of these are False"),
61  default=['doAssembleCcd', 'doFringe']
62  )
63  isrMandatoryStepsDarks = pexConfig.ListField(
64  dtype=str,
65  doc=("isr operations that must be performed for valid results when using darks. "
66  "Raises if any of these are False"),
67  default=['doAssembleCcd', 'doFringe']
68  )
69  isrForbiddenStepsFlats = pexConfig.ListField(
70  dtype=str,
71  doc=("isr operations that must NOT be performed for valid results when using flats."
72  " Raises if any of these are True"),
73  default=['doBrighterFatter', 'doUseOpticsTransmission',
74  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
75  )
76  isrForbiddenStepsDarks = pexConfig.ListField(
77  dtype=str,
78  doc=("isr operations that must NOT be performed for valid results when using darks."
79  " Raises if any of these are True"),
80  default=['doBrighterFatter', 'doUseOpticsTransmission',
81  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
82  )
83  isrDesirableSteps = pexConfig.ListField(
84  dtype=str,
85  doc=("isr operations that it is advisable to perform, but are not mission-critical."
86  " WARNs are logged for any of these found to be False."),
87  default=['doBias']
88  )
89  ccdKey = pexConfig.Field(
90  dtype=str,
91  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
92  default='ccd',
93  )
94  imageTypeKey = pexConfig.Field(
95  dtype=str,
96  doc="The key for the butler to use by which to check whether images are darks or flats",
97  default='imageType',
98  )
99  mode = pexConfig.ChoiceField(
100  doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?"
101  " If MASTER, a single visit number should be supplied, for which the corresponding master flat"
102  " and dark will be used. If VISITS, the list of visits will be used, treating the flats and "
103  " darks as appropriate, depending on their image types, as determined by their imageType from"
104  " config.imageTypeKey"),
105  dtype=str,
106  default="VISITS",
107  allowed={
108  "VISITS": "Calculate defects from a list of raw visits",
109  "MASTER": "Use the corresponding master calibs from the specified visit to measure defects",
110  }
111  )
112  nSigmaBright = pexConfig.Field(
113  dtype=float,
114  doc=("Number of sigma above mean for bright pixel detection. The default value was found to be",
115  " appropriate for some LSST sensors in DM-17490."),
116  default=4.8,
117  )
118  nSigmaDark = pexConfig.Field(
119  dtype=float,
120  doc=("Number of sigma below mean for dark pixel detection. The default value was found to be",
121  " appropriate for some LSST sensors in DM-17490."),
122  default=5.0,
123  )
124  nPixBorderUpDown = pexConfig.Field(
125  dtype=int,
126  doc="Number of pixels to exclude from top & bottom of image when looking for defects.",
127  default=7,
128  )
129  nPixBorderLeftRight = pexConfig.Field(
130  dtype=int,
131  doc="Number of pixels to exclude from left & right of image when looking for defects.",
132  default=7,
133  )
134  badOnAndOffPixelColumnThreshold = pexConfig.Field(
135  dtype=int,
136  doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ",
137  "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ",
138  "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ",
139  "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ",
140  "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ",
141  "non-bad pixels in BPC, an exception to the above is made and those consecutive ",
142  "'goodPixelColumnGapThreshold' are not marked as bad."),
143  default=50,
144  )
145  goodPixelColumnGapThreshold = pexConfig.Field(
146  dtype=int,
147  doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ",
148  "'badOnAndOffPixelColumnThreshold')."),
149  default=30,
150  )
151  edgesAsDefects = pexConfig.Field(
152  dtype=bool,
153  doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects."
154  " Normal treatment is to simply exclude this region from the defect finding, such that no"
155  " defect will be located there."),
156  default=False,
157  )
158  assertSameRun = pexConfig.Field(
159  dtype=bool,
160  doc=("Ensure that all visits are from the same run? Raises if this is not the case, or"
161  "if the run key isn't found."),
162  default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this.
163  )
164  ignoreFilters = pexConfig.Field(
165  dtype=bool,
166  doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input"
167  " images. Allows mixing of filters in the input flats. Set to False if you think"
168  " your defects might be chromatic and want to have registry support for varying"
169  " defects with respect to filter."),
170  default=True,
171  )
172  nullFilterName = pexConfig.Field(
173  dtype=str,
174  doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"),
175  default="NONE",
176  )
177  combinationMode = pexConfig.ChoiceField(
178  doc="Which types of defects to identify",
179  dtype=str,
180  default="FRACTION",
181  allowed={
182  "AND": "Logical AND the pixels found in each visit to form set ",
183  "OR": "Logical OR the pixels found in each visit to form set ",
184  "FRACTION": "Use pixels found in more than config.combinationFraction of visits ",
185  }
186  )
187  combinationFraction = pexConfig.RangeField(
188  dtype=float,
189  doc=("The fraction (0..1) of visits in which a pixel was found to be defective across"
190  " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use"
191  " mode AND to require pixel to appear in all images."),
192  default=0.7,
193  min=0,
194  max=1,
195  )
196  makePlots = pexConfig.Field(
197  dtype=bool,
198  doc=("Plot histograms for each visit for each amp (one plot per detector) and the final"
199  " defects overlaid on the sensor."),
200  default=False,
201  )
202  writeAs = pexConfig.ChoiceField(
203  doc="Write the output file as ASCII or FITS table",
204  dtype=str,
205  default="FITS",
206  allowed={
207  "ASCII": "Write the output as an ASCII file",
208  "FITS": "Write the output as an FITS table",
209  "BOTH": "Write the output as both a FITS table and an ASCII file",
210  }
211  )
212 
213 
214 class FindDefectsTask(pipeBase.CmdLineTask):
215  """Task for finding defects in sensors.
216 
217  The task has two modes of operation, defect finding in raws and in
218  master calibrations, which work as follows.
219 
220  Master calib defect finding
221  ----------------------------
222 
223  A single visit number is supplied, for which the corresponding flat & dark
224  will be used. This is because, at present at least, there is no way to pass
225  a calibration exposure ID from the command line to a command line task.
226 
227  The task retrieves the corresponding dark and flat exposures for the
228  supplied visit. If a flat is available the task will (be able to) look
229  for both bright and dark defects. If only a dark is found then only bright
230  defects will be sought.
231 
232  All pixels above/below the specified nSigma which lie with the specified
233  borders for flats/darks are identified as defects.
234 
235  Raw visit defect finding
236  ------------------------
237 
238  A list of exposure IDs are supplied for defect finding. The task will
239  detect bright pixels in the dark frames, if supplied, and bright & dark
240  pixels in the flats, if supplied, i.e. if you only supply darks you will
241  only be given bright defects. This is done automatically from the imageType
242  of the exposure, so the input exposure list can be a mix.
243 
244  As with the master calib detection, all pixels above/below the specified
245  nSigma which lie with the specified borders for flats/darks are identified
246  as defects. Then, a post-processing step is done to merge these detections,
247  with pixels appearing in a fraction [0..1] of the images are kept as defects
248  and those appearing below that occurrence-threshold are discarded.
249  """
250 
251  RunnerClass = SingleVisitListTaskRunner
252  ConfigClass = FindDefectsTaskConfig
253  _DefaultName = "findDefects"
254 
255  def __init__(self, *args, **kwargs):
256  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
257  self.makeSubtask("isrForFlats")
258  self.makeSubtask("isrForDarks")
259 
260  validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats,
261  self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps)
262  validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks,
263  self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps)
264  self.config.validate()
265  self.config.freeze()
266 
267  @classmethod
268  def _makeArgumentParser(cls):
269  """Augment argument parser for the FindDefectsTask."""
270  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
271  parser.add_argument("--visitList", dest="visitList", nargs="*",
272  help=("List of visits to use. Same for each detector."
273  " Uses the normal 0..10:3^234 syntax"))
274  parser.add_id_argument("--id", datasetType="newDefects",
275  ContainerClass=NonexistentDatasetTaskDataIdContainer,
276  help="The ccds to use, e.g. --id ccd=0..100")
277  return parser
278 
279  @pipeBase.timeMethod
280  def runDataRef(self, dataRef, visitList):
281  """Run the defect finding task.
282 
283  Find the defects, as described in the main task docstring, from a
284  dataRef and a list of visit(s).
285 
286  Parameters
287  ----------
288  dataRef : `lsst.daf.persistence.ButlerDataRef`
289  dataRef for the detector for the visits to be fit.
290  visitList : `list` [`int`]
291  List of visits to be processed. If config.mode == 'VISITS' then the
292  list of visits is used. If config.mode == 'MASTER' then the length
293  of visitList must be one, and the corresponding master calibrations
294  are used.
295 
296  Returns
297  -------
298  result : `lsst.pipe.base.Struct`
299  Result struct with Components:
300 
301  - ``defects`` : `lsst.meas.algorithms.Defect`
302  The defects found by the task.
303  - ``exitStatus`` : `int`
304  The exit code.
305  """
306 
307  detNum = dataRef.dataId[self.config.ccdKey]
308  self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum))
309 
310  defectLists = {'dark': [], 'flat': []}
311 
312  midTime = 0
313  filters = set()
314 
315  if self.config.mode == 'MASTER':
316  if len(visitList) > 1:
317  raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}")
318  dataRef.dataId['expId'] = visitList[0]
319 
320  for datasetType in defectLists.keys():
321  exp = dataRef.get(datasetType)
322  midTime += self._getMjd(exp)
323  filters.add(exp.getFilter().getName())
324  defects = self.findHotAndColdPixels(exp, datasetType)
325 
326  msg = "Found %s defects containing %s pixels in master %s"
327  self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType)
328  defectLists[datasetType].append(defects)
329  if self.config.makePlots:
330  self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType),
331  defects, datasetType)
332  midTime /= len(defectLists.keys())
333 
334  elif self.config.mode == 'VISITS':
335  butler = dataRef.getButler()
336 
337  if self.config.assertSameRun:
338  runs = self._getRunListFromVisits(butler, visitList)
339  if len(runs) != 1:
340  raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True")
341 
342  for visit in visitList:
343  imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'expId': visit})[0]
344  imageType = imageType.lower()
345  dataRef.dataId['expId'] = visit
346 
347  if imageType == 'flat': # note different isr tasks
348  exp = self.isrForFlats.runDataRef(dataRef).exposure
349  defects = self.findHotAndColdPixels(exp, imageType)
350  defectLists['flat'].append(defects)
351  midTime += self._getMjd(exp)
352  filters.add(exp.getFilter().getName())
353 
354  elif imageType == 'dark':
355  exp = self.isrForDarks.runDataRef(dataRef).exposure
356  defects = self.findHotAndColdPixels(exp, imageType)
357  defectLists['dark'].append(defects)
358  midTime += self._getMjd(exp)
359  filters.add(exp.getFilter().getName())
360 
361  else:
362  raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported")
363 
364  msg = "Found %s defects containing %s pixels in visit %s"
365  self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit)
366 
367  if self.config.makePlots:
368  self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType)
369 
370  midTime /= len(visitList)
371 
372  msg = "Combining %s defect sets from darks for detector %s"
373  self.log.info(msg, len(defectLists['dark']), detNum)
374  mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(),
375  self.config.combinationMode)
376  msg = "Combining %s defect sets from flats for detector %s"
377  self.log.info(msg, len(defectLists['flat']), detNum)
378  mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(),
379  self.config.combinationMode)
380 
381  msg = "Combining bright and dark defect sets for detector %s"
382  self.log.info(msg, detNum)
383  brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats]
384  allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR')
385 
386  self._writeData(dataRef, allDefects, midTime, filters)
387 
388  self.log.info("Finished finding defects in detector %s" % detNum)
389  return pipeBase.Struct(defects=allDefects, exitStatus=0)
390 
391  def _getNsigmaForPlot(self, imageType):
392  assert imageType in ['flat', 'dark']
393  nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark
394  return nSig
395 
396  @staticmethod
397  def _nPixFromDefects(defect):
398  """Count the number of pixels in a defect object."""
399  nPix = 0
400  for d in defect:
401  nPix += d.getBBox().getArea()
402  return nPix
403 
404  def _writeData(self, dataRef, defects, midTime, filters):
405  """Write the data out to the defect file.
406 
407  Parameters
408  ----------
409  dataRef : `lsst.daf.persistence.ButlerDataRef`
410  dataRef for the detector for defects to be written.
411  defects : `lsst.meas.algorithms.Defect`
412  The defects to be written.
413  """
414  date = dafBase.DateTime(midTime, dafBase.DateTime.MJD).toPython().isoformat()
415 
416  detName = self._getDetectorNameShort(dataRef)
417  instrumentName = self._getInstrumentName(dataRef)
418  detNum = self._getDetectorNumber(dataRef)
419  if not self.config.ignoreFilters:
420  filt = self._filterSetToFilterString(filters)
421  else:
422  filt = self.config.nullFilterName
423 
424  CALIB_ID = f"detectorName={detName} detector={detNum} calibDate={date} ccd={detNum} filter={filt}"
425  try:
426  raftName = self._getRaftName(dataRef)
427  CALIB_ID += f" raftName={raftName}"
428  except Exception:
429  pass
430 
431  now = dafBase.DateTime.now().toPython()
432  mdOriginal = defects.getMetadata()
433  mdSupplemental = {"INSTRUME": instrumentName,
434  "DETECTOR": dataRef.dataId['detector'],
435  "CALIBDATE": date,
436  "CALIB_ID": CALIB_ID,
437  "CALIB_CREATION_DATE": now.date().isoformat(),
438  "CALIB_CREATION_TIME": now.time().isoformat()}
439 
440  mdOriginal.update(mdSupplemental)
441 
442  # TODO: DM-23508 sort out the butler abuse from here-on down in Gen3
443  # defects should simply be butler.put()
444  templateFilename = dataRef.getUri(write=True) # does not guarantee that full path exists
445  baseDirName = os.path.dirname(templateFilename)
446  # ingest curated calibs demands detectorName is lowercase
447  detNameFull = self._getDetectorNameFull(dataRef)
448  dirName = os.path.join(baseDirName, instrumentName, "defects", detNameFull.lower())
449  if not os.path.exists(dirName):
450  os.makedirs(dirName)
451 
452  date += ".fits"
453  filename = os.path.join(dirName, date)
454 
455  msg = "Writing defects to %s in format: %s"
456  self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs)
457  if self.config.writeAs in ['FITS', 'BOTH']:
458  defects.writeFits(filename)
459  if self.config.writeAs in ['ASCII', 'BOTH']:
460  wroteTo = defects.writeText(filename)
461  assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0])
462  return
463 
464  @staticmethod
465  def _filterSetToFilterString(filters):
466  return "~".join([f for f in filters])
467 
468  @staticmethod
469  def _getDetectorNumber(dataRef):
470  """The detector's integer identifier."""
471  dataRefDetNum = dataRef.dataId['detector']
472  camera = dataRef.get('camera')
473  detectorDetNum = camera[dataRef.dataId['detector']].getId()
474  assert dataRefDetNum == detectorDetNum
475  return dataRefDetNum
476 
477  @staticmethod
478  def _getInstrumentName(dataRef):
479  camera = dataRef.get('camera')
480  return camera.getName()
481 
482  @staticmethod
483  def _getDetectorNameFull(dataRef):
484  """The detector's self-reported full name, e.g. R12_S01."""
485  camera = dataRef.get('camera')
486  return camera[dataRef.dataId['detector']].getName()
487 
488  @staticmethod
489  def _getDetectorNameShort(dataRef):
490  """The detectorName per the butler, e.g. slot name, e.g. S12."""
491  butler = dataRef.getButler()
492  detectorName = butler.queryMetadata('raw', ['detectorName'], dataRef.dataId)[0]
493  return detectorName
494 
495  @staticmethod
496  def _getRaftName(dataRef):
497  """The detectorName per the butler, e.g. slot name, e.g. S12."""
498  butler = dataRef.getButler()
499  raftName = butler.queryMetadata('raw', ['raftName'], dataRef.dataId)[0]
500  return raftName
501 
502  @staticmethod
503  def _getMjd(exp, timescale=dafBase.DateTime.UTC):
504  vi = exp.getInfo().getVisitInfo()
505  dateObs = vi.getDate()
506  mjd = dateObs.get(dafBase.DateTime.MJD)
507  return mjd
508 
509  @staticmethod
510  def _getRunListFromVisits(butler, visitList):
511  """Return the set of runs for the visits in visitList."""
512  runs = set()
513  for visit in visitList:
514  runs.add(butler.queryMetadata('raw', 'run', dataId={'expId': visit})[0])
515  return runs
516 
517  def _postProcessDefectSets(self, defectList, imageDimensions, mode):
518  """Combine a list of defects to make a single defect object.
519 
520  AND, OR or use percentage of visits in which defects appear
521  depending on config.
522 
523  Parameters
524  ----------
525  defectList : `list` [`lsst.meas.algorithms.Defect`]
526  The lList of defects to merge.
527  imageDimensions : `tuple` [`int`]
528  The size of the image.
529  mode : `str`
530  The combination mode to use, either 'AND', 'OR' or 'FRACTION'
531 
532  Returns
533  -------
534  defects : `lsst.meas.algorithms.Defect`
535  The defect set resulting from the merge.
536  """
537  # so that empty lists can be passed in for input data
538  # where only flats or darks are supplied
539  if defectList == []:
540  return []
541 
542  if len(defectList) == 1: # single input - no merging to do
543  return defectList[0]
544 
545  sumImage = afwImage.MaskedImageF(imageDimensions)
546  for defects in defectList:
547  for defect in defects:
548  sumImage.image[defect.getBBox()] += 1
549  sumImage /= len(defectList)
550 
551  nDetected = len(np.where(sumImage.image.array > 0)[0])
552  self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected)
553 
554  if mode == 'OR': # must appear in any
555  indices = np.where(sumImage.image.array > 0)
556  else:
557  if mode == 'AND': # must appear in all
558  threshold = 1
559  elif mode == 'FRACTION':
560  threshold = self.config.combinationFraction
561  else:
562  raise RuntimeError(f"Got unsupported combinationMode {mode}")
563  indices = np.where(sumImage.image.array >= threshold)
564 
565  BADBIT = sumImage.mask.getPlaneBitMask('BAD')
566  sumImage.mask.array[indices] |= BADBIT
567 
568  self.log.info("Post-merge %s pixels marked as defects" % len(indices[0]))
569 
570  if self.config.edgesAsDefects:
571  self.log.info("Masking edge pixels as defects in addition to previously identified defects")
572  self._setEdgeBits(sumImage, 'BAD')
573 
574  defects = measAlg.Defects.fromMask(sumImage, 'BAD')
575  return defects
576 
577  @staticmethod
578  def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"):
579  """Return the number of non-bad pixels in the image."""
580  nPixels = maskedIm.mask.array.size
581  nBad = countMaskedPixels(maskedIm, badMaskString)
582  return nPixels - nBad
583 
584  def findHotAndColdPixels(self, exp, imageType, setMask=False):
585  """Find hot and cold pixels in an image.
586 
587  Using config-defined thresholds on a per-amp basis, mask pixels
588  that are nSigma above threshold in dark frames (hot pixels),
589  or nSigma away from the clipped mean in flats (hot & cold pixels).
590 
591  Parameters
592  ----------
593  exp : `lsst.afw.image.exposure.Exposure`
594  The exposure in which to find defects.
595  imageType : `str`
596  The image type, either 'dark' or 'flat'.
597  setMask : `bool`
598  If true, update exp with hot and cold pixels.
599  hot: DETECTED
600  cold: DETECTED_NEGATIVE
601 
602  Returns
603  -------
604  defects : `lsst.meas.algorithms.Defect`
605  The defects found in the image.
606  """
607  assert imageType in ['flat', 'dark']
608 
609  self._setEdgeBits(exp)
610  maskedIm = exp.maskedImage
611 
612  # the detection polarity for afwDetection, True for positive,
613  # False for negative, and therefore True for darks as they only have
614  # bright pixels, and both for flats, as they have bright and dark pix
615  polarities = {'dark': [True], 'flat': [True, False]}[imageType]
616 
617  footprintList = []
618 
619  for amp in exp.getDetector():
620  ampImg = maskedIm[amp.getBBox()].clone()
621 
622  # crop ampImage depending on where the amp lies in the image
623  if self.config.nPixBorderLeftRight:
624  if ampImg.getX0() == 0:
625  ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL]
626  else:
627  ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL]
628  if self.config.nPixBorderUpDown:
629  if ampImg.getY0() == 0:
630  ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL]
631  else:
632  ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL]
633 
634  if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels
635  continue
636 
637  ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue()
638 
639  mergedSet = None
640  for polarity in polarities:
641  nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark
642  threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity)
643 
644  footprintSet = afwDetection.FootprintSet(ampImg, threshold)
645  if setMask:
646  footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE"))
647 
648  if mergedSet is None:
649  mergedSet = footprintSet
650  else:
651  mergedSet.merge(footprintSet)
652 
653  footprintList += mergedSet.getFootprints()
654 
655  defects = measAlg.Defects.fromFootprintList(footprintList)
656  defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects)
657 
658  return defects
659 
661  """Mask blocks in a column if there are on-and-off bad pixels
662 
663  If there's a column with on and off bad pixels, mask all the pixels in between,
664  except if there is a large enough gap of consecutive good pixels between two
665  bad pixels in the column.
666 
667  Parameters
668  ---------
669  defects: `lsst.meas.algorithms.Defect`
670  The defects found in the image so far
671 
672  Returns
673  ------
674  defects: `lsst.meas.algorithms.Defect`
675  If the number of bad pixels in a column is not larger or equal than
676  self.config.badPixelColumnThreshold, the iput list is returned. Otherwise,
677  the defects list returned will include boxes that mask blocks of on-and-of
678  pixels.
679  """
680  # Get the (x, y) values of each bad pixel in amp.
681  coordinates = []
682  for defect in defects:
683  bbox = defect.getBBox()
684  x0, y0 = bbox.getMinX(), bbox.getMinY()
685  deltaX0, deltaY0 = bbox.getDimensions()
686  for j in np.arange(y0, y0+deltaY0):
687  for i in np.arange(x0, x0 + deltaX0):
688  coordinates.append((i, j))
689 
690  x, y = [], []
691  for coordinatePair in coordinates:
692  x.append(coordinatePair[0])
693  y.append(coordinatePair[1])
694 
695  x = np.array(x)
696  y = np.array(y)
697  # Find the defects with same "x" (vertical) coordinate (column).
698  unique, counts = np.unique(x, return_counts=True)
699  multipleX = []
700  for (a, b) in zip(unique, counts):
701  if b >= self.config.badOnAndOffPixelColumnThreshold:
702  multipleX.append(a)
703  if len(multipleX) != 0:
704  defects = self._markBlocksInBadColumn(x, y, multipleX, defects)
705 
706  return defects
707 
708  def _markBlocksInBadColumn(self, x, y, multipleX, defects):
709  """Mask blocks in a column if number of on-and-off bad pixels is above threshold.
710 
711  This function is called if the number of on-and-off bad pixels in a column
712  is larger or equal than self.config.badOnAndOffPixelColumnThreshold.
713 
714  Parameters
715  ---------
716  x: list
717  Lower left x coordinate of defect box. x coordinate is along the short axis if amp.
718 
719  y: list
720  Lower left y coordinate of defect box. x coordinate is along the long axis if amp.
721 
722  multipleX: list
723  List of x coordinates in amp. with multiple bad pixels (i.e., columns with defects).
724 
725  defects: `lsst.meas.algorithms.Defect`
726  The defcts found in the image so far
727 
728  Returns
729  -------
730  defects: `lsst.meas.algorithms.Defect`
731  The defects list returned that will include boxes that mask blocks
732  of on-and-of pixels.
733  """
734  with defects.bulk_update():
735  goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold
736  for x0 in multipleX:
737  index = np.where(x == x0)
738  multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence.
739  minY, maxY = np.min(multipleY), np.max(multipleY)
740  # Next few lines: don't mask pixels in column if gap of good pixels between
741  # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'.
742  diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0]
743  if len(diffIndex) != 0:
744  limits = [minY] # put the minimum first
745  for gapIndex in diffIndex:
746  limits.append(multipleY[gapIndex])
747  limits.append(multipleY[gapIndex+1])
748  limits.append(maxY) # maximum last
749  assert len(limits)%2 == 0, 'limits is even by design, but check anyways'
750  for i in np.arange(0, len(limits)-1, 2):
751  s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1]))
752  defects.append(s)
753  else: # No gap is large enough
754  s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY))
755  defects.append(s)
756  return defects
757 
758  def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'):
759  """Set edge bits on an exposure or maskedImage.
760 
761  Raises
762  ------
763  TypeError
764  Raised if parameter ``exposureOrMaskedImage`` is an invalid type.
765  """
766  if isinstance(exposureOrMaskedImage, afwImage.Exposure):
767  mi = exposureOrMaskedImage.maskedImage
768  elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage):
769  mi = exposureOrMaskedImage
770  else:
771  t = type(exposureOrMaskedImage)
772  raise TypeError(f"Function supports exposure or maskedImage but not {t}")
773 
774  MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet)
775  if self.config.nPixBorderLeftRight:
776  mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT
777  mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT
778  if self.config.nPixBorderUpDown:
779  mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT
780  mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT
781 
782  def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover
783  """Plot the defects and pixel histograms.
784 
785  Parameters
786  ----------
787  dataRef : `lsst.daf.persistence.ButlerDataRef`
788  dataRef for the detector.
789  exp : `lsst.afw.image.exposure.Exposure`
790  The exposure in which the defects were found.
791  visit : `int`
792  The visit number.
793  nSig : `float`
794  The number of sigma used for detection
795  defects : `lsst.meas.algorithms.Defect`
796  The defects to plot.
797  imageType : `str`
798  The type of image, either 'dark' or 'flat'.
799 
800  Currently only for LSST sensors. Plots are written to the path
801  given by the butler for the ``cpPipePlotRoot`` dataset type.
802  """
803  import matplotlib.pyplot as plt
804  from matplotlib.backends.backend_pdf import PdfPages
805 
806  afwDisplay.setDefaultBackend("matplotlib")
807  plt.interactive(False) # seems to need reasserting here
808 
809  dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
810  if not os.path.exists(dirname):
811  os.makedirs(dirname)
812 
813  detNum = exp.getDetector().getId()
814  nAmps = len(exp.getDetector())
815 
816  if self.config.mode == "MASTER":
817  filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf"
818  elif self.config.mode == "VISITS":
819  filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf"
820 
821  filenameFull = os.path.join(dirname, filename)
822 
823  with warnings.catch_warnings():
824  msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure."
825  warnings.filterwarnings("ignore", message=msg)
826  with PdfPages(filenameFull) as pdfPages:
827  if nAmps == 16:
828  self._plotAmpHistogram(dataRef, exp, visit, nSig)
829  pdfPages.savefig()
830 
831  self._plotDefects(exp, visit, defects, imageType)
832  pdfPages.savefig()
833  self.log.info("Wrote plot(s) to %s" % filenameFull)
834 
835  def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover
836  """Plot the defects found by the task.
837 
838  Parameters
839  ----------
840  exp : `lsst.afw.image.exposure.Exposure`
841  The exposure in which the defects were found.
842  visit : `int`
843  The visit number.
844  defects : `lsst.meas.algorithms.Defect`
845  The defects to plot.
846  imageType : `str`
847  The type of image, either 'dark' or 'flat'.
848  """
849  expCopy = exp.clone() # we mess with the copy later, so make a clone
850  del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes
851  maskedIm = expCopy.maskedImage
852 
853  defects.maskPixels(expCopy.maskedImage, "BAD")
854  detector = expCopy.getDetector()
855 
856  disp = afwDisplay.Display(0, reopenPlot=True, dpi=200)
857 
858  if imageType == "flat": # set each amp image to have a mean of 1.00
859  for amp in detector:
860  ampIm = maskedIm.image[amp.getBBox()]
861  ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1
862 
863  mpDict = maskedIm.mask.getMaskPlaneDict()
864  for plane in mpDict.keys():
865  if plane in ['BAD']:
866  continue
867  disp.setMaskPlaneColor(plane, afwDisplay.IGNORE)
868 
869  disp.scale('asinh', 'zscale')
870  disp.setMaskTransparency(80)
871  disp.setMaskPlaneColor("BAD", afwDisplay.RED)
872 
873  disp.setImageColormap('gray')
874  title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}"
875  f", Type: {imageType}, visit: {visit}")
876  disp.mtv(maskedIm, title=title)
877 
878  cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp)
879 
880  def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover
881  """
882  Make a histogram of the distribution of pixel values for each amp.
883 
884  The main image data histogram is plotted in blue. Edge pixels,
885  if masked, are in red. Note that masked edge pixels do not contribute
886  to the underflow and overflow numbers.
887 
888  Note that this currently only supports the 16-amp LSST detectors.
889 
890  Parameters
891  ----------
892  dataRef : `lsst.daf.persistence.ButlerDataRef`
893  dataRef for the detector.
894  exp : `lsst.afw.image.exposure.Exposure`
895  The exposure in which the defects were found.
896  visit : `int`
897  The visit number.
898  nSigmaUsed : `float`
899  The number of sigma used for detection
900  """
901  import matplotlib.pyplot as plt
902 
903  detector = exp.getDetector()
904 
905  if len(detector) != 16:
906  raise RuntimeError("Plotting currently only supported for 16 amp detectors")
907  fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10))
908 
909  expTime = exp.getInfo().getVisitInfo().getExposureTime()
910 
911  for (amp, a) in zip(reversed(detector), ax.flatten()):
912  mi = exp.maskedImage[amp.getBBox()]
913 
914  # normalize by expTime as we plot in ADU/s and don't always work with master calibs
915  mi.image.array /= expTime
916  stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP)
917  mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
918  # Get array of pixels
919  EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE")
920  imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten()
921  edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten()
922 
923  thrUpper = mean + nSigmaUsed*sigma
924  thrLower = mean - nSigmaUsed*sigma
925 
926  nRight = len(imgData[imgData > thrUpper])
927  nLeft = len(imgData[imgData < thrLower])
928 
929  nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used
930  leftEdge = mean - nsig * nSigmaUsed*sigma
931  rightEdge = mean + nsig * nSigmaUsed*sigma
932  nbins = np.linspace(leftEdge, rightEdge, 1000)
933  ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red')
934  y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue')
935 
936  # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram
937  nOverflow = len(imgData[imgData > rightEdge])
938  nUnderflow = len(imgData[imgData < leftEdge])
939 
940  # Put v-lines and textboxes in
941  a.axvline(thrUpper, c='k')
942  a.axvline(thrLower, c='k')
943  msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}"
944  a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11)
945  msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}"
946  a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5)
947 
948  # set axis limits and scales
949  a.set_ylim([1., 1.7*np.max(y)])
950  lPlot, rPlot = a.get_xlim()
951  a.set_xlim(np.array([lPlot, rPlot]))
952  a.set_yscale('log')
953  a.set_xlabel("ADU/s")
954 
955  return
lsst.cp.pipe.defects.FindDefectsTask.findHotAndColdPixels
def findHotAndColdPixels(self, exp, imageType, setMask=False)
Definition: defects.py:584
lsst.cp.pipe.defects.FindDefectsTask._getNumGoodPixels
def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA")
Definition: defects.py:578
lsst::afw::image
lsst.cp.pipe.defects.FindDefectsTask
Definition: defects.py:214
lsst.cp.pipe.utils.validateIsrConfig
def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, checkTrim=True, logName=None)
Definition: utils.py:496
lsst.cp.pipe.defects.FindDefectsTaskConfig
Definition: defects.py:46
lsst.cp.pipe.defects.FindDefectsTask._postProcessDefectSets
def _postProcessDefectSets(self, defectList, imageDimensions, mode)
Definition: defects.py:517
lsst.cp.pipe.defects.FindDefectsTask._plot
def _plot(self, dataRef, exp, visit, nSig, defects, imageType)
Definition: defects.py:782
lsst.cp.pipe.defects.FindDefectsTask.__init__
def __init__(self, *args, **kwargs)
Definition: defects.py:255
lsst.cp.pipe.defects.FindDefectsTask._setEdgeBits
def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE')
Definition: defects.py:758
lsst.cp.pipe.defects.FindDefectsTask.maskBlocksIfIntermitentBadPixelsInColumn
def maskBlocksIfIntermitentBadPixelsInColumn(self, defects)
Definition: defects.py:660
lsst.cp.pipe.defects.FindDefectsTask._getRunListFromVisits
def _getRunListFromVisits(butler, visitList)
Definition: defects.py:510
lsst::afw
lsst::afw::display
lsst.cp.pipe.defects.FindDefectsTask.runDataRef
def runDataRef(self, dataRef, visitList)
Definition: defects.py:280
lsst.cp.pipe.defects.FindDefectsTask._filterSetToFilterString
def _filterSetToFilterString(filters)
Definition: defects.py:465
lsst.cp.pipe.utils.countMaskedPixels
def countMaskedPixels(maskedIm, maskPlane)
Definition: utils.py:158
lsst.cp.pipe.defects.FindDefectsTask._getDetectorNameShort
def _getDetectorNameShort(dataRef)
Definition: defects.py:489
lsst.cp.pipe.defects.FindDefectsTask._plotDefects
def _plotDefects(self, exp, visit, defects, imageType)
Definition: defects.py:835
lsst::pex::config
lsst.cp.pipe.defects.FindDefectsTask._writeData
def _writeData(self, dataRef, defects, midTime, filters)
Definition: defects.py:404
lsst.cp.pipe.defects.FindDefectsTask._getInstrumentName
def _getInstrumentName(dataRef)
Definition: defects.py:478
lsst::afw::detection
lsst.cp.pipe.defects.FindDefectsTask._getNsigmaForPlot
def _getNsigmaForPlot(self, imageType)
Definition: defects.py:391
lsst::geom
lsst::daf::base
lsst::ip::isr
lsst::afw::math
Point< int, 2 >
lsst::geom::Box2I
lsst.cp.pipe.defects.FindDefectsTask._getDetectorNameFull
def _getDetectorNameFull(dataRef)
Definition: defects.py:483
lsst.cp.pipe.defects.FindDefectsTask._getRaftName
def _getRaftName(dataRef)
Definition: defects.py:496
lsst.cp.pipe.defects.FindDefectsTask._nPixFromDefects
def _nPixFromDefects(defect)
Definition: defects.py:397
lsst::pipe::base
lsst::meas::algorithms
lsst.cp.pipe.defects.FindDefectsTask._getMjd
def _getMjd(exp, timescale=dafBase.DateTime.UTC)
Definition: defects.py:503
lsst.cp.pipe.defects.FindDefectsTask._DefaultName
string _DefaultName
Definition: defects.py:253
lsst.cp.pipe.defects.FindDefectsTask._getDetectorNumber
def _getDetectorNumber(dataRef)
Definition: defects.py:469
lsst.cp.pipe.defects.FindDefectsTask._markBlocksInBadColumn
def _markBlocksInBadColumn(self, x, y, multipleX, defects)
Definition: defects.py:708
lsst.cp.pipe.defects.FindDefectsTask._plotAmpHistogram
def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed)
Definition: defects.py:880