lsst.ip.isr  16.0-24-g587d83f
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defect
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "RunIsrTask"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="ExposureU",
78  dimensions=["Instrument", "Exposure", "Detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["Instrument", "ExposureRange"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["Instrument", "ExposureRange", "Detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["Instrument", "ExposureRange", "Detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["Instrument", "PhysicalFilter", "ExposureRange", "Detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["Instrument", "ExposureRange"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="Catalog",
120  dimensions=["Instrument", "ExposureRange", "Detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["Instrument", "ExposureRange"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["Instrument", "PhysicalFilter", "ExposureRange"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["Instrument", "ExposureRange", "Detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["Instrument"],
149  )
150 
151  # output datasets
152  outputExposure = pipeBase.OutputDatasetField(
153  doc="Output ISR processed exposure.",
154  name="postISRCCD",
155  scalar=True,
156  storageClass="ExposureF",
157  dimensions=["Instrument", "Visit", "Detector"],
158  )
159  outputOssThumbnail = pipeBase.OutputDatasetField(
160  doc="Output Overscan-subtracted thumbnail image.",
161  name="OssThumb",
162  scalar=True,
163  storageClass="Thumbnail",
164  dimensions=["Instrument", "Visit", "Detector"],
165  )
166  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167  doc="Output flat-corrected thumbnail image.",
168  name="FlattenedThumb",
169  scalar=True,
170  storageClass="TextStorage",
171  dimensions=["Instrument", "Visit", "Detector"],
172  )
173 
174  quantum = pipeBase.QuantumConfig(
175  dimensions=["Visit", "Detector", "Instrument"],
176  )
177 
178 
179  datasetType = pexConfig.Field(
180  dtype=str,
181  doc="Dataset type for input data; users will typically leave this alone, "
182  "but camera-specific ISR tasks will override it",
183  default="raw",
184  )
185 
186  fallbackFilterName = pexConfig.Field(
187  dtype=str,
188  doc="Fallback default filter name for calibrations.",
189  optional=True
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
195  )
196  fwhm = pexConfig.Field(
197  dtype=float,
198  doc="FWHM of PSF in arcseconds.",
199  default=1.0,
200  )
201  qa = pexConfig.ConfigField(
202  dtype=isrQa.IsrQaConfig,
203  doc="QA related configuration options.",
204  )
205 
206  # Image conversion configuration
207  doConvertIntToFloat = pexConfig.Field(
208  dtype=bool,
209  doc="Convert integer raw images to floating point values?",
210  default=True,
211  )
212 
213  # Saturated pixel handling.
214  doSaturation = pexConfig.Field(
215  dtype=bool,
216  doc="Mask saturated pixels?",
217  default=True,
218  )
219  saturatedMaskName = pexConfig.Field(
220  dtype=str,
221  doc="Name of mask plane to use in saturation detection and interpolation",
222  default="SAT",
223  )
224  saturation = pexConfig.Field(
225  dtype=float,
226  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
227  default=float("NaN"),
228  )
229  growSaturationFootprintSize = pexConfig.Field(
230  dtype=int,
231  doc="Number of pixels by which to grow the saturation footprints",
232  default=1,
233  )
234 
235  # Suspect pixel handling.
236  doSuspect = pexConfig.Field(
237  dtype=bool,
238  doc="Mask suspect pixels?",
239  default=True,
240  )
241  suspectMaskName = pexConfig.Field(
242  dtype=str,
243  doc="Name of mask plane to use for suspect pixels",
244  default="SUSPECT",
245  )
246  numEdgeSuspect = pexConfig.Field(
247  dtype=int,
248  doc="Number of edge pixels to be flagged as untrustworthy.",
249  default=0,
250  )
251 
252  # Initial masking options.
253  doSetBadRegions = pexConfig.Field(
254  dtype=bool,
255  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
256  default=True,
257  )
258  badStatistic = pexConfig.ChoiceField(
259  dtype=str,
260  doc="How to estimate the average value for BAD regions.",
261  default='MEANCLIP',
262  allowed={
263  "MEANCLIP": "Correct using the (clipped) mean of good data",
264  "MEDIAN": "Correct using the median of the good data",
265  },
266  )
267 
268  # Overscan subtraction configuration.
269  doOverscan = pexConfig.Field(
270  dtype=bool,
271  doc="Do overscan subtraction?",
272  default=True,
273  )
274  overscanFitType = pexConfig.ChoiceField(
275  dtype=str,
276  doc="The method for fitting the overscan bias level.",
277  default='MEDIAN',
278  allowed={
279  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
280  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
281  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
282  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
283  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
284  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
285  "MEAN": "Correct using the mean of the overscan region",
286  "MEANCLIP": "Correct using a clipped mean of the overscan region",
287  "MEDIAN": "Correct using the median of the overscan region",
288  },
289  )
290  overscanOrder = pexConfig.Field(
291  dtype=int,
292  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
293  "or number of spline knots if overscan fit type is a spline."),
294  default=1,
295  )
296  overscanNumSigmaClip = pexConfig.Field(
297  dtype=float,
298  doc="Rejection threshold (sigma) for collapsing overscan before fit",
299  default=3.0,
300  )
301  overscanIsInt = pexConfig.Field(
302  dtype=bool,
303  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
304  default=True,
305  )
306  overscanNumLeadingColumnsToSkip = pexConfig.Field(
307  dtype=int,
308  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
309  default=0,
310  )
311  overscanNumTrailingColumnsToSkip = pexConfig.Field(
312  dtype=int,
313  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
314  default=0,
315  )
316  overscanMaxDev = pexConfig.Field(
317  dtype=float,
318  doc="Maximum deviation from the median for overscan",
319  default=1000.0, check=lambda x: x > 0
320  )
321  overscanBiasJump = pexConfig.Field(
322  dtype=bool,
323  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
324  default=False,
325  )
326  overscanBiasJumpKeyword = pexConfig.Field(
327  dtype=str,
328  doc="Header keyword containing information about devices.",
329  default="NO_SUCH_KEY",
330  )
331  overscanBiasJumpDevices = pexConfig.ListField(
332  dtype=str,
333  doc="List of devices that need piecewise overscan correction.",
334  default=(),
335  )
336  overscanBiasJumpLocation = pexConfig.Field(
337  dtype=int,
338  doc="Location of bias jump along y-axis.",
339  default=0,
340  )
341 
342  # Amplifier to CCD assembly configuration
343  doAssembleCcd = pexConfig.Field(
344  dtype=bool,
345  default=True,
346  doc="Assemble amp-level exposures into a ccd-level exposure?"
347  )
348  assembleCcd = pexConfig.ConfigurableField(
349  target=AssembleCcdTask,
350  doc="CCD assembly task",
351  )
352 
353  # General calibration configuration.
354  doAssembleIsrExposures = pexConfig.Field(
355  dtype=bool,
356  default=False,
357  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
358  )
359  doTrimToMatchCalib = pexConfig.Field(
360  dtype=bool,
361  default=False,
362  doc="Trim raw data to match calibration bounding boxes?"
363  )
364 
365  # Bias subtraction.
366  doBias = pexConfig.Field(
367  dtype=bool,
368  doc="Apply bias frame correction?",
369  default=True,
370  )
371  biasDataProductName = pexConfig.Field(
372  dtype=str,
373  doc="Name of the bias data product",
374  default="bias",
375  )
376 
377  # Variance construction
378  doVariance = pexConfig.Field(
379  dtype=bool,
380  doc="Calculate variance?",
381  default=True
382  )
383  gain = pexConfig.Field(
384  dtype=float,
385  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
386  default=float("NaN"),
387  )
388  readNoise = pexConfig.Field(
389  dtype=float,
390  doc="The read noise to use if no Detector is present in the Exposure",
391  default=0.0,
392  )
393  doEmpiricalReadNoise = pexConfig.Field(
394  dtype=bool,
395  default=False,
396  doc="Calculate empirical read noise instead of value from AmpInfo data?"
397  )
398 
399  # Linearization.
400  doLinearize = pexConfig.Field(
401  dtype=bool,
402  doc="Correct for nonlinearity of the detector's response?",
403  default=True,
404  )
405 
406  # Crosstalk.
407  doCrosstalk = pexConfig.Field(
408  dtype=bool,
409  doc="Apply intra-CCD crosstalk correction?",
410  default=False,
411  )
412  doCrosstalkBeforeAssemble = pexConfig.Field(
413  dtype=bool,
414  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
415  default=True,
416  )
417  crosstalk = pexConfig.ConfigurableField(
418  target=CrosstalkTask,
419  doc="Intra-CCD crosstalk correction",
420  )
421 
422  # Masking option prior to brighter-fatter.
423  doWidenSaturationTrails = pexConfig.Field(
424  dtype=bool,
425  doc="Widen bleed trails based on their width?",
426  default=True
427  )
428 
429  # Brighter-Fatter correction.
430  doBrighterFatter = pexConfig.Field(
431  dtype=bool,
432  default=False,
433  doc="Apply the brighter fatter correction"
434  )
435  brighterFatterLevel = pexConfig.ChoiceField(
436  dtype=str,
437  default="DETECTOR",
438  doc="The level at which to correct for brighter-fatter.",
439  allowed={
440  "AMP": "Every amplifier treated separately.",
441  "DETECTOR": "One kernel per detector",
442  }
443  )
444  brighterFatterKernelFile = pexConfig.Field(
445  dtype=str,
446  default='',
447  doc="Kernel file used for the brighter fatter correction"
448  )
449  brighterFatterMaxIter = pexConfig.Field(
450  dtype=int,
451  default=10,
452  doc="Maximum number of iterations for the brighter fatter correction"
453  )
454  brighterFatterThreshold = pexConfig.Field(
455  dtype=float,
456  default=1000,
457  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
458  " absolute value of the difference between the current corrected image and the one"
459  " from the previous iteration summed over all the pixels."
460  )
461  brighterFatterApplyGain = pexConfig.Field(
462  dtype=bool,
463  default=True,
464  doc="Should the gain be applied when applying the brighter fatter correction?"
465  )
466 
467  # Defect and bad pixel correction options.
468  doDefect = pexConfig.Field(
469  dtype=bool,
470  doc="Apply correction for CCD defects, e.g. hot pixels?",
471  default=True,
472  )
473  doSaturationInterpolation = pexConfig.Field(
474  dtype=bool,
475  doc="Perform interpolation over pixels masked as saturated?",
476  default=True,
477  )
478  numEdgeSuspect = pexConfig.Field(
479  dtype=int,
480  doc="Number of edge pixels to be flagged as untrustworthy.",
481  default=0,
482  )
483 
484  # Dark subtraction.
485  doDark = pexConfig.Field(
486  dtype=bool,
487  doc="Apply dark frame correction?",
488  default=True,
489  )
490  darkDataProductName = pexConfig.Field(
491  dtype=str,
492  doc="Name of the dark data product",
493  default="dark",
494  )
495 
496  # Camera-specific stray light removal.
497  doStrayLight = pexConfig.Field(
498  dtype=bool,
499  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
500  default=False,
501  )
502  strayLight = pexConfig.ConfigurableField(
503  target=StrayLightTask,
504  doc="y-band stray light correction"
505  )
506 
507  # Flat correction.
508  doFlat = pexConfig.Field(
509  dtype=bool,
510  doc="Apply flat field correction?",
511  default=True,
512  )
513  flatDataProductName = pexConfig.Field(
514  dtype=str,
515  doc="Name of the flat data product",
516  default="flat",
517  )
518  flatScalingType = pexConfig.ChoiceField(
519  dtype=str,
520  doc="The method for scaling the flat on the fly.",
521  default='USER',
522  allowed={
523  "USER": "Scale by flatUserScale",
524  "MEAN": "Scale by the inverse of the mean",
525  "MEDIAN": "Scale by the inverse of the median",
526  },
527  )
528  flatUserScale = pexConfig.Field(
529  dtype=float,
530  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
531  default=1.0,
532  )
533  doTweakFlat = pexConfig.Field(
534  dtype=bool,
535  doc="Tweak flats to match observed amplifier ratios?",
536  default=False
537  )
538 
539  # Amplifier normalization based on gains instead of using flats configuration.
540  doApplyGains = pexConfig.Field(
541  dtype=bool,
542  doc="Correct the amplifiers for their gains instead of applying flat correction",
543  default=False,
544  )
545  normalizeGains = pexConfig.Field(
546  dtype=bool,
547  doc="Normalize all the amplifiers in each CCD to have the same median value.",
548  default=False,
549  )
550 
551  # Fringe correction.
552  doFringe = pexConfig.Field(
553  dtype=bool,
554  doc="Apply fringe correction?",
555  default=True,
556  )
557  fringe = pexConfig.ConfigurableField(
558  target=FringeTask,
559  doc="Fringe subtraction task",
560  )
561  fringeAfterFlat = pexConfig.Field(
562  dtype=bool,
563  doc="Do fringe subtraction after flat-fielding?",
564  default=True,
565  )
566 
567  # NAN pixel interpolation option.
568  doNanInterpAfterFlat = pexConfig.Field(
569  dtype=bool,
570  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
571  "also have to interpolate them before flat-fielding."),
572  default=False,
573  )
574 
575  # Distortion model application.
576  doAddDistortionModel = pexConfig.Field(
577  dtype=bool,
578  doc="Apply a distortion model based on camera geometry to the WCS?",
579  default=True,
580  )
581 
582  # Initial CCD-level background statistics options.
583  doMeasureBackground = pexConfig.Field(
584  dtype=bool,
585  doc="Measure the background level on the reduced image?",
586  default=False,
587  )
588 
589  # Camera-specific masking configuration.
590  doCameraSpecificMasking = pexConfig.Field(
591  dtype=bool,
592  doc="Mask camera-specific bad regions?",
593  default=False,
594  )
595  masking = pexConfig.ConfigurableField(
596  target=MaskingTask,
597  doc="Masking task."
598  )
599 
600  # Default photometric calibration options.
601  fluxMag0T1 = pexConfig.DictField(
602  keytype=str,
603  itemtype=float,
604  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
605  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
606  ))
607  )
608  defaultFluxMag0T1 = pexConfig.Field(
609  dtype=float,
610  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
611  default=pow(10.0, 0.4*28.0)
612  )
613 
614  # Vignette correction configuration.
615  doVignette = pexConfig.Field(
616  dtype=bool,
617  doc="Apply vignetting parameters?",
618  default=False,
619  )
620  vignette = pexConfig.ConfigurableField(
621  target=VignetteTask,
622  doc="Vignetting task.",
623  )
624 
625  # Transmission curve configuration.
626  doAttachTransmissionCurve = pexConfig.Field(
627  dtype=bool,
628  default=False,
629  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
630  )
631  doUseOpticsTransmission = pexConfig.Field(
632  dtype=bool,
633  default=True,
634  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
635  )
636  doUseFilterTransmission = pexConfig.Field(
637  dtype=bool,
638  default=True,
639  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
640  )
641  doUseSensorTransmission = pexConfig.Field(
642  dtype=bool,
643  default=True,
644  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
645  )
646  doUseAtmosphereTransmission = pexConfig.Field(
647  dtype=bool,
648  default=True,
649  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
650  )
651 
652  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
653  doWrite = pexConfig.Field(
654  dtype=bool,
655  doc="Persist postISRCCD?",
656  default=True,
657  )
658 
659  def validate(self):
660  super().validate()
661  if self.doFlat and self.doApplyGains:
662  raise ValueError("You may not specify both doFlat and doApplyGains")
663 
664 
665 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
666  r"""Apply common instrument signature correction algorithms to a raw frame.
667 
668  The process for correcting imaging data is very similar from
669  camera to camera. This task provides a vanilla implementation of
670  doing these corrections, including the ability to turn certain
671  corrections off if they are not needed. The inputs to the primary
672  method, `run()`, are a raw exposure to be corrected and the
673  calibration data products. The raw input is a single chip sized
674  mosaic of all amps including overscans and other non-science
675  pixels. The method `runDataRef()` identifies and defines the
676  calibration data products, and is intended for use by a
677  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
678  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
679  subclassed for different camera, although the most camera specific
680  methods have been split into subtasks that can be redirected
681  appropriately.
682 
683  The __init__ method sets up the subtasks for ISR processing, using
684  the defaults from `lsst.ip.isr`.
685 
686  Parameters
687  ----------
688  args : `list`
689  Positional arguments passed to the Task constructor. None used at this time.
690  kwargs : `dict`, optional
691  Keyword arguments passed on to the Task constructor. None used at this time.
692  """
693  ConfigClass = IsrTaskConfig
694  _DefaultName = "isr"
695 
696  def __init__(self, **kwargs):
697  super().__init__(**kwargs)
698  self.makeSubtask("assembleCcd")
699  self.makeSubtask("crosstalk")
700  self.makeSubtask("strayLight")
701  self.makeSubtask("fringe")
702  self.makeSubtask("masking")
703  self.makeSubtask("vignette")
704 
705  @classmethod
706  def getInputDatasetTypes(cls, config):
707  inputTypeDict = super().getInputDatasetTypes(config)
708 
709  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
710  # need because the configuration tells us we will not be bothering with the
711  # correction that uses that IDT.
712  if config.doBias is not True:
713  inputTypeDict.pop("bias", None)
714  if config.doLinearize is not True:
715  inputTypeDict.pop("linearizer", None)
716  if config.doCrosstalk is not True:
717  inputTypeDict.pop("crosstalkSources", None)
718  if config.doBrighterFatter is not True:
719  inputTypeDict.pop("bfKernel", None)
720  if config.doDefect is not True:
721  inputTypeDict.pop("defects", None)
722  if config.doDark is not True:
723  inputTypeDict.pop("dark", None)
724  if config.doFlat is not True:
725  inputTypeDict.pop("flat", None)
726  if config.doAttachTransmissionCurve is not True:
727  inputTypeDict.pop("opticsTransmission", None)
728  inputTypeDict.pop("filterTransmission", None)
729  inputTypeDict.pop("sensorTransmission", None)
730  inputTypeDict.pop("atmosphereTransmission", None)
731  if config.doUseOpticsTransmission is not True:
732  inputTypeDict.pop("opticsTransmission", None)
733  if config.doUseFilterTransmission is not True:
734  inputTypeDict.pop("filterTransmission", None)
735  if config.doUseSensorTransmission is not True:
736  inputTypeDict.pop("sensorTransmission", None)
737  if config.doUseAtmosphereTransmission is not True:
738  inputTypeDict.pop("atmosphereTransmission", None)
739 
740  return inputTypeDict
741 
742  @classmethod
743  def getOutputDatasetTypes(cls, config):
744  outputTypeDict = super().getOutputDatasetTypes(config)
745 
746  if config.qa.doThumbnailOss is not True:
747  outputTypeDict.pop("outputOssThumbnail", None)
748  if config.qa.doThumbnailFlattened is not True:
749  outputTypeDict.pop("outputFlattenedThumbnail", None)
750  if config.doWrite is not True:
751  outputTypeDict.pop("outputExposure", None)
752 
753  return outputTypeDict
754 
755  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
756  try:
757  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
758  except Exception as e:
759  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
760 
761  inputData['isGen3'] = True
762 
763  if self.config.doLinearize is True:
764  if 'linearizer' not in inputData.keys():
765  detector = inputData['camera'][inputData['detectorNum']]
766  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
767  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
768 
769  if inputData['defects'] is not None:
770  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
771  # masking expects a list of defects defined by their bounding box
772  defectList = []
773 
774  for r in inputData['defects']:
775  bbox = afwGeom.BoxI(afwGeom.PointI(r.get("x0"), r.get("y0")),
776  afwGeom.ExtentI(r.get("width"), r.get("height")))
777  defectList.append(Defect(bbox))
778 
779  inputData['defects'] = defectList
780 
781  # Broken: DM-17169
782  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
783  # fixed for non-HSC cameras in the future.
784  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
785  # if self.config.doCrosstalk else None)
786 
787  # Broken: DM-17152
788  # Fringes are not tested to be handled correctly by Gen3 butler.
789  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
790  # assembler=self.assembleCcd
791  # if self.config.doAssembleIsrExposures else None)
792  # if self.config.doFringe and
793  # self.fringe.checkFilter(inputData['ccdExposure'])
794  # else pipeBase.Struct(fringes=None))
795 
796  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
797 
798  def makeDatasetType(self, dsConfig):
799  return super().makeDatasetType(dsConfig)
800 
801  def readIsrData(self, dataRef, rawExposure):
802  """!Retrieve necessary frames for instrument signature removal.
803 
804  Pre-fetching all required ISR data products limits the IO
805  required by the ISR. Any conflict between the calibration data
806  available and that needed for ISR is also detected prior to
807  doing processing, allowing it to fail quickly.
808 
809  Parameters
810  ----------
811  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
812  Butler reference of the detector data to be processed
813  rawExposure : `afw.image.Exposure`
814  The raw exposure that will later be corrected with the
815  retrieved calibration data; should not be modified in this
816  method.
817 
818  Returns
819  -------
820  result : `lsst.pipe.base.Struct`
821  Result struct with components (which may be `None`):
822  - ``bias``: bias calibration frame (`afw.image.Exposure`)
823  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
824  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
825  - ``dark``: dark calibration frame (`afw.image.Exposure`)
826  - ``flat``: flat calibration frame (`afw.image.Exposure`)
827  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
828  - ``defects``: list of defects (`list`)
829  - ``fringes``: `lsst.pipe.base.Struct` with components:
830  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
831  - ``seed``: random seed derived from the ccdExposureId for random
832  number generator (`uint32`)
833  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
834  A ``TransmissionCurve`` that represents the throughput of the optics,
835  to be evaluated in focal-plane coordinates.
836  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
837  A ``TransmissionCurve`` that represents the throughput of the filter
838  itself, to be evaluated in focal-plane coordinates.
839  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
840  A ``TransmissionCurve`` that represents the throughput of the sensor
841  itself, to be evaluated in post-assembly trimmed detector coordinates.
842  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
843  A ``TransmissionCurve`` that represents the throughput of the
844  atmosphere, assumed to be spatially constant.
845 
846  """
847  ccd = rawExposure.getDetector()
848  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
849  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
850  if self.config.doBias else None)
851  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
852  linearizer = (dataRef.get("linearizer", immediate=True)
853  if self.doLinearize(ccd) else None)
854  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
855  if self.config.doCrosstalk else None)
856  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
857  if self.config.doDark else None)
858  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
859  if self.config.doFlat else None)
860  brighterFatterKernel = (dataRef.get("bfKernel")
861  if self.config.doBrighterFatter else None)
862  defectList = (dataRef.get("defects")
863  if self.config.doDefect else None)
864  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
865  if self.config.doAssembleIsrExposures else None)
866  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
867  else pipeBase.Struct(fringes=None))
868 
869  if self.config.doAttachTransmissionCurve:
870  opticsTransmission = (dataRef.get("transmission_optics")
871  if self.config.doUseOpticsTransmission else None)
872  filterTransmission = (dataRef.get("transmission_filter")
873  if self.config.doUseFilterTransmission else None)
874  sensorTransmission = (dataRef.get("transmission_sensor")
875  if self.config.doUseSensorTransmission else None)
876  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
877  if self.config.doUseAtmosphereTransmission else None)
878  else:
879  opticsTransmission = None
880  filterTransmission = None
881  sensorTransmission = None
882  atmosphereTransmission = None
883 
884  # Struct should include only kwargs to run()
885  return pipeBase.Struct(bias=biasExposure,
886  linearizer=linearizer,
887  crosstalkSources=crosstalkSources,
888  dark=darkExposure,
889  flat=flatExposure,
890  bfKernel=brighterFatterKernel,
891  defects=defectList,
892  fringes=fringeStruct,
893  opticsTransmission=opticsTransmission,
894  filterTransmission=filterTransmission,
895  sensorTransmission=sensorTransmission,
896  atmosphereTransmission=atmosphereTransmission,
897  )
898 
899  @pipeBase.timeMethod
900  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
901  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
902  opticsTransmission=None, filterTransmission=None,
903  sensorTransmission=None, atmosphereTransmission=None,
904  detectorNum=None, isGen3=False
905  ):
906  """!Perform instrument signature removal on an exposure.
907 
908  Steps included in the ISR processing, in order performed, are:
909  - saturation and suspect pixel masking
910  - overscan subtraction
911  - CCD assembly of individual amplifiers
912  - bias subtraction
913  - variance image construction
914  - linearization of non-linear response
915  - crosstalk masking
916  - brighter-fatter correction
917  - dark subtraction
918  - fringe correction
919  - stray light subtraction
920  - flat correction
921  - masking of known defects and camera specific features
922  - vignette calculation
923  - appending transmission curve and distortion model
924 
925  Parameters
926  ----------
927  ccdExposure : `lsst.afw.image.Exposure`
928  The raw exposure that is to be run through ISR. The
929  exposure is modified by this method.
930  camera : `lsst.afw.cameraGeom.Camera`, optional
931  The camera geometry for this exposure. Used to select the
932  distortion model appropriate for this data.
933  bias : `lsst.afw.image.Exposure`, optional
934  Bias calibration frame.
935  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
936  Functor for linearization.
937  crosstalkSources : `list`, optional
938  List of possible crosstalk sources.
939  dark : `lsst.afw.image.Exposure`, optional
940  Dark calibration frame.
941  flat : `lsst.afw.image.Exposure`, optional
942  Flat calibration frame.
943  bfKernel : `numpy.ndarray`, optional
944  Brighter-fatter kernel.
945  defects : `list`, optional
946  List of defects.
947  fringes : `lsst.pipe.base.Struct`, optional
948  Struct containing the fringe correction data, with
949  elements:
950  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
951  - ``seed``: random seed derived from the ccdExposureId for random
952  number generator (`uint32`)
953  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
954  A ``TransmissionCurve`` that represents the throughput of the optics,
955  to be evaluated in focal-plane coordinates.
956  filterTransmission : `lsst.afw.image.TransmissionCurve`
957  A ``TransmissionCurve`` that represents the throughput of the filter
958  itself, to be evaluated in focal-plane coordinates.
959  sensorTransmission : `lsst.afw.image.TransmissionCurve`
960  A ``TransmissionCurve`` that represents the throughput of the sensor
961  itself, to be evaluated in post-assembly trimmed detector coordinates.
962  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
963  A ``TransmissionCurve`` that represents the throughput of the
964  atmosphere, assumed to be spatially constant.
965  detectorNum : `int`, optional
966  The integer number for the detector to process.
967  isGen3 : bool, optional
968  Flag this call to run() as using the Gen3 butler environment.
969 
970  Returns
971  -------
972  result : `lsst.pipe.base.Struct`
973  Result struct with component:
974  - ``exposure`` : `afw.image.Exposure`
975  The fully ISR corrected exposure.
976  - ``outputExposure`` : `afw.image.Exposure`
977  An alias for `exposure`
978  - ``ossThumb`` : `numpy.ndarray`
979  Thumbnail image of the exposure after overscan subtraction.
980  - ``flattenedThumb`` : `numpy.ndarray`
981  Thumbnail image of the exposure after flat-field correction.
982 
983  Raises
984  ------
985  RuntimeError
986  Raised if a configuration option is set to True, but the
987  required calibration data has not been specified.
988 
989  Notes
990  -----
991  The current processed exposure can be viewed by setting the
992  appropriate lsstDebug entries in the `debug.display`
993  dictionary. The names of these entries correspond to some of
994  the IsrTaskConfig Boolean options, with the value denoting the
995  frame to use. The exposure is shown inside the matching
996  option check and after the processing of that step has
997  finished. The steps with debug points are:
998 
999  doAssembleCcd
1000  doBias
1001  doCrosstalk
1002  doBrighterFatter
1003  doDark
1004  doFringe
1005  doStrayLight
1006  doFlat
1007 
1008  In addition, setting the "postISRCCD" entry displays the
1009  exposure after all ISR processing has finished.
1010 
1011  """
1012 
1013  if isGen3 is True:
1014  # Gen3 currently cannot automatically do configuration overrides.
1015  # DM-15257 looks to discuss this issue.
1016 
1017  self.config.doFringe = False
1018 
1019  # Configure input exposures;
1020  if detectorNum is None:
1021  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1022 
1023  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1024  bias = self.ensureExposure(bias, camera, detectorNum)
1025  dark = self.ensureExposure(dark, camera, detectorNum)
1026  flat = self.ensureExposure(flat, camera, detectorNum)
1027  else:
1028  if isinstance(ccdExposure, ButlerDataRef):
1029  return self.runDataRef(ccdExposure)
1030 
1031  ccd = ccdExposure.getDetector()
1032 
1033  if not ccd:
1034  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1035  ccd = [FakeAmp(ccdExposure, self.config)]
1036 
1037  # Validate Input
1038  if self.config.doBias and bias is None:
1039  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1040  if self.doLinearize(ccd) and linearizer is None:
1041  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1042  if self.config.doBrighterFatter and bfKernel is None:
1043  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1044  if self.config.doDark and dark is None:
1045  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1046  if fringes is None:
1047  fringes = pipeBase.Struct(fringes=None)
1048  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1049  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1050  if self.config.doFlat and flat is None:
1051  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1052  if self.config.doDefect and defects is None:
1053  raise RuntimeError("Must supply defects if config.doDefect=True.")
1054  if self.config.doAddDistortionModel and camera is None:
1055  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1056 
1057  # Begin ISR processing.
1058  if self.config.doConvertIntToFloat:
1059  self.log.info("Converting exposure to floating point values")
1060  ccdExposure = self.convertIntToFloat(ccdExposure)
1061 
1062  # Amplifier level processing.
1063  overscans = []
1064  for amp in ccd:
1065  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1066  if ccdExposure.getBBox().contains(amp.getBBox()):
1067  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1068  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1069 
1070  if self.config.doOverscan and not badAmp:
1071  # Overscan correction on amp-by-amp basis.
1072  overscanResults = self.overscanCorrection(ccdExposure, amp)
1073  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1074  if self.config.qa is not None and self.config.qa.saveStats is True:
1075  if isinstance(overscanResults.overscanFit, float):
1076  qaMedian = overscanResults.overscanFit
1077  qaStdev = float("NaN")
1078  else:
1079  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1080  afwMath.MEDIAN | afwMath.STDEVCLIP)
1081  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1082  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1083 
1084  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1085  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1086  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1087  (amp.getName(), qaMedian, qaStdev))
1088  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1089  else:
1090  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1091  overscanResults = None
1092 
1093  overscans.append(overscanResults if overscanResults is not None else None)
1094  else:
1095  self.log.info("Skipped OSCAN")
1096 
1097  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1098  self.log.info("Applying crosstalk correction.")
1099  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1100  self.debugView(ccdExposure, "doCrosstalk")
1101 
1102  if self.config.doAssembleCcd:
1103  self.log.info("Assembling CCD from amplifiers")
1104  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1105 
1106  if self.config.expectWcs and not ccdExposure.getWcs():
1107  self.log.warn("No WCS found in input exposure")
1108  self.debugView(ccdExposure, "doAssembleCcd")
1109 
1110  ossThumb = None
1111  if self.config.qa.doThumbnailOss:
1112  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1113 
1114  if self.config.doBias:
1115  self.log.info("Applying bias correction.")
1116  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1117  trimToFit=self.config.doTrimToMatchCalib)
1118  self.debugView(ccdExposure, "doBias")
1119 
1120  if self.config.doVariance:
1121  for amp, overscanResults in zip(ccd, overscans):
1122  if ccdExposure.getBBox().contains(amp.getBBox()):
1123  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1124  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1125  if overscanResults is not None:
1126  self.updateVariance(ampExposure, amp,
1127  overscanImage=overscanResults.overscanImage)
1128  else:
1129  self.updateVariance(ampExposure, amp,
1130  overscanImage=None)
1131  if self.config.qa is not None and self.config.qa.saveStats is True:
1132  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1133  afwMath.MEDIAN | afwMath.STDEVCLIP)
1134  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1135  qaStats.getValue(afwMath.MEDIAN))
1136  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1137  qaStats.getValue(afwMath.STDEVCLIP))
1138  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1139  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1140  qaStats.getValue(afwMath.STDEVCLIP)))
1141 
1142  if self.doLinearize(ccd):
1143  self.log.info("Applying linearizer.")
1144  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1145 
1146  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1147  self.log.info("Applying crosstalk correction.")
1148  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1149  self.debugView(ccdExposure, "doCrosstalk")
1150 
1151  if self.config.doWidenSaturationTrails:
1152  self.log.info("Widening saturation trails.")
1153  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1154 
1155  interpolationDone = False
1156  if self.config.doBrighterFatter:
1157  # We need to apply flats and darks before we can interpolate, and we
1158  # need to interpolate before we do B-F, but we do B-F without the
1159  # flats and darks applied so we can work in units of electrons or holes.
1160  # This context manager applies and then removes the darks and flats.
1161  with self.flatContext(ccdExposure, flat, dark):
1162  if self.config.doDefect:
1163  self.maskAndInterpDefect(ccdExposure, defects)
1164 
1165  if self.config.doSaturationInterpolation:
1166  self.saturationInterpolation(ccdExposure)
1167 
1168  self.maskAndInterpNan(ccdExposure)
1169  interpolationDone = True
1170 
1171  if self.config.brighterFatterLevel == 'DETECTOR':
1172  kernelElement = bfKernel
1173  else:
1174  # TODO: DM-15631 for implementing this
1175  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
1176  self.log.info("Applying brighter fatter correction.")
1177  isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1178  self.config.brighterFatterMaxIter,
1179  self.config.brighterFatterThreshold,
1180  self.config.brighterFatterApplyGain,
1181  )
1182  self.debugView(ccdExposure, "doBrighterFatter")
1183 
1184  if self.config.doDark:
1185  self.log.info("Applying dark correction.")
1186  self.darkCorrection(ccdExposure, dark)
1187  self.debugView(ccdExposure, "doDark")
1188 
1189  if self.config.doFringe and not self.config.fringeAfterFlat:
1190  self.log.info("Applying fringe correction before flat.")
1191  self.fringe.run(ccdExposure, **fringes.getDict())
1192  self.debugView(ccdExposure, "doFringe")
1193 
1194  if self.config.doStrayLight:
1195  self.log.info("Applying stray light correction.")
1196  self.strayLight.run(ccdExposure)
1197  self.debugView(ccdExposure, "doStrayLight")
1198 
1199  if self.config.doFlat:
1200  self.log.info("Applying flat correction.")
1201  self.flatCorrection(ccdExposure, flat)
1202  self.debugView(ccdExposure, "doFlat")
1203 
1204  if self.config.doApplyGains:
1205  self.log.info("Applying gain correction instead of flat.")
1206  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1207 
1208  if self.config.doDefect and not interpolationDone:
1209  self.log.info("Masking and interpolating defects.")
1210  self.maskAndInterpDefect(ccdExposure, defects)
1211 
1212  if self.config.doSaturation and not interpolationDone:
1213  self.log.info("Interpolating saturated pixels.")
1214  self.saturationInterpolation(ccdExposure)
1215 
1216  if self.config.doNanInterpAfterFlat or not interpolationDone:
1217  self.log.info("Masking and interpolating NAN value pixels.")
1218  self.maskAndInterpNan(ccdExposure)
1219 
1220  if self.config.doFringe and self.config.fringeAfterFlat:
1221  self.log.info("Applying fringe correction after flat.")
1222  self.fringe.run(ccdExposure, **fringes.getDict())
1223 
1224  if self.config.doSetBadRegions:
1225  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1226  if badPixelCount > 0:
1227  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1228 
1229  flattenedThumb = None
1230  if self.config.qa.doThumbnailFlattened:
1231  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1232 
1233  if self.config.doCameraSpecificMasking:
1234  self.log.info("Masking regions for camera specific reasons.")
1235  self.masking.run(ccdExposure)
1236 
1237  self.roughZeroPoint(ccdExposure)
1238 
1239  if self.config.doVignette:
1240  self.log.info("Constructing Vignette polygon.")
1241  self.vignettePolygon = self.vignette.run(ccdExposure)
1242 
1243  if self.config.vignette.doWriteVignettePolygon:
1244  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1245 
1246  if self.config.doAttachTransmissionCurve:
1247  self.log.info("Adding transmission curves.")
1248  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1249  filterTransmission=filterTransmission,
1250  sensorTransmission=sensorTransmission,
1251  atmosphereTransmission=atmosphereTransmission)
1252 
1253  if self.config.doAddDistortionModel:
1254  self.log.info("Adding a distortion model to the WCS.")
1255  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1256 
1257  if self.config.doMeasureBackground:
1258  self.log.info("Measuring background level:")
1259  self.measureBackground(ccdExposure, self.config.qa)
1260 
1261  if self.config.qa is not None and self.config.qa.saveStats is True:
1262  for amp in ccd:
1263  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1264  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1265  afwMath.MEDIAN | afwMath.STDEVCLIP)
1266  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1267  qaStats.getValue(afwMath.MEDIAN))
1268  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1269  qaStats.getValue(afwMath.STDEVCLIP))
1270  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1271  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1272  qaStats.getValue(afwMath.STDEVCLIP)))
1273 
1274  self.debugView(ccdExposure, "postISRCCD")
1275 
1276  return pipeBase.Struct(
1277  exposure=ccdExposure,
1278  ossThumb=ossThumb,
1279  flattenedThumb=flattenedThumb,
1280 
1281  outputExposure=ccdExposure,
1282  outputOssThumbnail=ossThumb,
1283  outputFlattenedThumbnail=flattenedThumb,
1284  )
1285 
1286  @pipeBase.timeMethod
1287  def runDataRef(self, sensorRef):
1288  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1289 
1290  This method contains the `CmdLineTask` interface to the ISR
1291  processing. All IO is handled here, freeing the `run()` method
1292  to manage only pixel-level calculations. The steps performed
1293  are:
1294  - Read in necessary detrending/isr/calibration data.
1295  - Process raw exposure in `run()`.
1296  - Persist the ISR-corrected exposure as "postISRCCD" if
1297  config.doWrite=True.
1298 
1299  Parameters
1300  ----------
1301  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1302  DataRef of the detector data to be processed
1303 
1304  Returns
1305  -------
1306  result : `lsst.pipe.base.Struct`
1307  Result struct with component:
1308  - ``exposure`` : `afw.image.Exposure`
1309  The fully ISR corrected exposure.
1310 
1311  Raises
1312  ------
1313  RuntimeError
1314  Raised if a configuration option is set to True, but the
1315  required calibration data does not exist.
1316 
1317  """
1318  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1319 
1320  ccdExposure = sensorRef.get(self.config.datasetType)
1321 
1322  camera = sensorRef.get("camera")
1323  if camera is None and self.config.doAddDistortionModel:
1324  raise RuntimeError("config.doAddDistortionModel is True "
1325  "but could not get a camera from the butler")
1326  isrData = self.readIsrData(sensorRef, ccdExposure)
1327 
1328  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1329 
1330  if self.config.doWrite:
1331  sensorRef.put(result.exposure, "postISRCCD")
1332  if result.ossThumb is not None:
1333  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1334  if result.flattenedThumb is not None:
1335  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1336 
1337  return result
1338 
1339  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1340  """!Retrieve a calibration dataset for removing instrument signature.
1341 
1342  Parameters
1343  ----------
1344 
1345  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1346  DataRef of the detector data to find calibration datasets
1347  for.
1348  datasetType : `str`
1349  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1350  immediate : `Bool`
1351  If True, disable butler proxies to enable error handling
1352  within this routine.
1353 
1354  Returns
1355  -------
1356  exposure : `lsst.afw.image.Exposure`
1357  Requested calibration frame.
1358 
1359  Raises
1360  ------
1361  RuntimeError
1362  Raised if no matching calibration frame can be found.
1363  """
1364  try:
1365  exp = dataRef.get(datasetType, immediate=immediate)
1366  except Exception as exc1:
1367  if not self.config.fallbackFilterName:
1368  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1369  try:
1370  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1371  except Exception as exc2:
1372  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1373  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1374  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1375 
1376  if self.config.doAssembleIsrExposures:
1377  exp = self.assembleCcd.assembleCcd(exp)
1378  return exp
1379 
1380  def ensureExposure(self, inputExp, camera, detectorNum):
1381  """Ensure that the data returned by Butler is a fully constructed exposure.
1382 
1383  ISR requires exposure-level image data for historical reasons, so if we did
1384  not recieve that from Butler, construct it from what we have, modifying the
1385  input in place.
1386 
1387  Parameters
1388  ----------
1389  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1390  `lsst.afw.image.ImageF`
1391  The input data structure obtained from Butler.
1392  camera : `lsst.afw.cameraGeom.camera`
1393  The camera associated with the image. Used to find the appropriate
1394  detector.
1395  detectorNum : `int`
1396  The detector this exposure should match.
1397 
1398  Returns
1399  -------
1400  inputExp : `lsst.afw.image.Exposure`
1401  The re-constructed exposure, with appropriate detector parameters.
1402 
1403  Raises
1404  ------
1405  TypeError
1406  Raised if the input data cannot be used to construct an exposure.
1407  """
1408  if isinstance(inputExp, afwImage.DecoratedImageU):
1409  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1410  elif isinstance(inputExp, afwImage.ImageF):
1411  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1412  elif isinstance(inputExp, afwImage.MaskedImageF):
1413  inputExp = afwImage.makeExposure(inputExp)
1414  elif isinstance(inputExp, afwImage.Exposure):
1415  pass
1416  else:
1417  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1418 
1419  if inputExp.getDetector() is None:
1420  inputExp.setDetector(camera[detectorNum])
1421 
1422  return inputExp
1423 
1424  def convertIntToFloat(self, exposure):
1425  """Convert exposure image from uint16 to float.
1426 
1427  If the exposure does not need to be converted, the input is
1428  immediately returned. For exposures that are converted to use
1429  floating point pixels, the variance is set to unity and the
1430  mask to zero.
1431 
1432  Parameters
1433  ----------
1434  exposure : `lsst.afw.image.Exposure`
1435  The raw exposure to be converted.
1436 
1437  Returns
1438  -------
1439  newexposure : `lsst.afw.image.Exposure`
1440  The input ``exposure``, converted to floating point pixels.
1441 
1442  Raises
1443  ------
1444  RuntimeError
1445  Raised if the exposure type cannot be converted to float.
1446 
1447  """
1448  if isinstance(exposure, afwImage.ExposureF):
1449  # Nothing to be done
1450  return exposure
1451  if not hasattr(exposure, "convertF"):
1452  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1453 
1454  newexposure = exposure.convertF()
1455  newexposure.variance[:] = 1
1456  newexposure.mask[:] = 0x0
1457 
1458  return newexposure
1459 
1460  def maskAmplifier(self, ccdExposure, amp, defects):
1461  """Identify bad amplifiers, saturated and suspect pixels.
1462 
1463  Parameters
1464  ----------
1465  ccdExposure : `lsst.afw.image.Exposure`
1466  Input exposure to be masked.
1467  amp : `lsst.afw.table.AmpInfoCatalog`
1468  Catalog of parameters defining the amplifier on this
1469  exposure to mask.
1470  defects : `list`
1471  List of defects. Used to determine if the entire
1472  amplifier is bad.
1473 
1474  Returns
1475  -------
1476  badAmp : `Bool`
1477  If this is true, the entire amplifier area is covered by
1478  defects and unusable.
1479 
1480  """
1481  maskedImage = ccdExposure.getMaskedImage()
1482 
1483  badAmp = False
1484 
1485  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1486  # comparison with current defects definition.
1487  if defects is not None:
1488  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1489 
1490  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1491  # association with pixels in current ccdExposure).
1492  if badAmp:
1493  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1494  afwImage.PARENT)
1495  maskView = dataView.getMask()
1496  maskView |= maskView.getPlaneBitMask("BAD")
1497  del maskView
1498  return badAmp
1499 
1500  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1501  # Saturation and suspect pixels can be masked now, though.
1502  limits = dict()
1503  if self.config.doSaturation and not badAmp:
1504  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1505  if self.config.doSuspect and not badAmp:
1506  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1507 
1508  for maskName, maskThreshold in limits.items():
1509  if not math.isnan(maskThreshold):
1510  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1511  isrFunctions.makeThresholdMask(
1512  maskedImage=dataView,
1513  threshold=maskThreshold,
1514  growFootprints=0,
1515  maskName=maskName
1516  )
1517 
1518  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1519  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1520  afwImage.PARENT)
1521  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1522  self.config.suspectMaskName])
1523  if numpy.all(maskView.getArray() & maskVal > 0):
1524  badAmp = True
1525 
1526  return badAmp
1527 
1528  def overscanCorrection(self, ccdExposure, amp):
1529  """Apply overscan correction in place.
1530 
1531  This method does initial pixel rejection of the overscan
1532  region. The overscan can also be optionally segmented to
1533  allow for discontinuous overscan responses to be fit
1534  separately. The actual overscan subtraction is performed by
1535  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1536  which is called here after the amplifier is preprocessed.
1537 
1538  Parameters
1539  ----------
1540  ccdExposure : `lsst.afw.image.Exposure`
1541  Exposure to have overscan correction performed.
1542  amp : `lsst.afw.table.AmpInfoCatalog`
1543  The amplifier to consider while correcting the overscan.
1544 
1545  Returns
1546  -------
1547  overscanResults : `lsst.pipe.base.Struct`
1548  Result struct with components:
1549  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1550  Value or fit subtracted from the amplifier image data.
1551  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1552  Value or fit subtracted from the overscan image data.
1553  - ``overscanImage`` : `lsst.afw.image.Image`
1554  Image of the overscan region with the overscan
1555  correction applied. This quantity is used to estimate
1556  the amplifier read noise empirically.
1557 
1558  Raises
1559  ------
1560  RuntimeError
1561  Raised if the ``amp`` does not contain raw pixel information.
1562 
1563  See Also
1564  --------
1565  lsst.ip.isr.isrFunctions.overscanCorrection
1566  """
1567  if not amp.getHasRawInfo():
1568  raise RuntimeError("This method must be executed on an amp with raw information.")
1569 
1570  if amp.getRawHorizontalOverscanBBox().isEmpty():
1571  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1572  return None
1573 
1574  statControl = afwMath.StatisticsControl()
1575  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1576 
1577  # Determine the bounding boxes
1578  dataBBox = amp.getRawDataBBox()
1579  oscanBBox = amp.getRawHorizontalOverscanBBox()
1580  dx0 = 0
1581  dx1 = 0
1582 
1583  prescanBBox = amp.getRawPrescanBBox()
1584  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1585  dx0 += self.config.overscanNumLeadingColumnsToSkip
1586  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1587  else:
1588  dx0 += self.config.overscanNumTrailingColumnsToSkip
1589  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1590 
1591  # Determine if we need to work on subregions of the amplifier and overscan.
1592  imageBBoxes = []
1593  overscanBBoxes = []
1594 
1595  if ((self.config.overscanBiasJump and
1596  self.config.overscanBiasJumpLocation) and
1597  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1598  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1599  self.config.overscanBiasJumpDevices)):
1600  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1601  yLower = self.config.overscanBiasJumpLocation
1602  yUpper = dataBBox.getHeight() - yLower
1603  else:
1604  yUpper = self.config.overscanBiasJumpLocation
1605  yLower = dataBBox.getHeight() - yUpper
1606 
1607  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1608  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1609  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1610  afwGeom.Extent2I(dx0, 0),
1611  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1612  yLower)))
1613 
1614  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1615  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1616  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1617  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1618  yUpper)))
1619  else:
1620  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1621  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1622  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1623  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1624  oscanBBox.getHeight())))
1625 
1626  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1627  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1628  ampImage = ccdExposure.maskedImage[imageBBox]
1629  overscanImage = ccdExposure.maskedImage[overscanBBox]
1630 
1631  overscanArray = overscanImage.image.array
1632  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1633  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1634  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1635 
1636  statControl = afwMath.StatisticsControl()
1637  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1638 
1639  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1640  overscanImage=overscanImage,
1641  fitType=self.config.overscanFitType,
1642  order=self.config.overscanOrder,
1643  collapseRej=self.config.overscanNumSigmaClip,
1644  statControl=statControl,
1645  overscanIsInt=self.config.overscanIsInt
1646  )
1647 
1648  # Measure average overscan levels and record them in the metadata
1649  levelStat = afwMath.MEDIAN
1650  sigmaStat = afwMath.STDEVCLIP
1651 
1652  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1653  self.config.qa.flatness.nIter)
1654  metadata = ccdExposure.getMetadata()
1655  ampNum = amp.getName()
1656  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1657  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1658  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1659  else:
1660  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1661  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1662  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1663 
1664  return overscanResults
1665 
1666  def updateVariance(self, ampExposure, amp, overscanImage=None):
1667  """Set the variance plane using the amplifier gain and read noise
1668 
1669  The read noise is calculated from the ``overscanImage`` if the
1670  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1671  the value from the amplifier data is used.
1672 
1673  Parameters
1674  ----------
1675  ampExposure : `lsst.afw.image.Exposure`
1676  Exposure to process.
1677  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1678  Amplifier detector data.
1679  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1680  Image of overscan, required only for empirical read noise.
1681 
1682  See also
1683  --------
1684  lsst.ip.isr.isrFunctions.updateVariance
1685  """
1686  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1687  gain = amp.getGain()
1688 
1689  if math.isnan(gain):
1690  gain = 1.0
1691  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1692  elif gain <= 0:
1693  patchedGain = 1.0
1694  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1695  (amp.getName(), gain, patchedGain))
1696  gain = patchedGain
1697 
1698  if self.config.doEmpiricalReadNoise and overscanImage is None:
1699  self.log.info("Overscan is none for EmpiricalReadNoise")
1700 
1701  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1702  stats = afwMath.StatisticsControl()
1703  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1704  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1705  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1706  else:
1707  readNoise = amp.getReadNoise()
1708 
1709  isrFunctions.updateVariance(
1710  maskedImage=ampExposure.getMaskedImage(),
1711  gain=gain,
1712  readNoise=readNoise,
1713  )
1714 
1715  def darkCorrection(self, exposure, darkExposure, invert=False):
1716  """!Apply dark correction in place.
1717 
1718  Parameters
1719  ----------
1720  exposure : `lsst.afw.image.Exposure`
1721  Exposure to process.
1722  darkExposure : `lsst.afw.image.Exposure`
1723  Dark exposure of the same size as ``exposure``.
1724  invert : `Bool`, optional
1725  If True, re-add the dark to an already corrected image.
1726 
1727  Raises
1728  ------
1729  RuntimeError
1730  Raised if either ``exposure`` or ``darkExposure`` do not
1731  have their dark time defined.
1732 
1733  See Also
1734  --------
1735  lsst.ip.isr.isrFunctions.darkCorrection
1736  """
1737  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1738  if math.isnan(expScale):
1739  raise RuntimeError("Exposure darktime is NAN")
1740  if darkExposure.getInfo().getVisitInfo() is not None:
1741  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1742  else:
1743  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1744  # so getDarkTime() does not exist.
1745  darkScale = 1.0
1746 
1747  if math.isnan(darkScale):
1748  raise RuntimeError("Dark calib darktime is NAN")
1749  isrFunctions.darkCorrection(
1750  maskedImage=exposure.getMaskedImage(),
1751  darkMaskedImage=darkExposure.getMaskedImage(),
1752  expScale=expScale,
1753  darkScale=darkScale,
1754  invert=invert,
1755  trimToFit=self.config.doTrimToMatchCalib
1756  )
1757 
1758  def doLinearize(self, detector):
1759  """!Check if linearization is needed for the detector cameraGeom.
1760 
1761  Checks config.doLinearize and the linearity type of the first
1762  amplifier.
1763 
1764  Parameters
1765  ----------
1766  detector : `lsst.afw.cameraGeom.Detector`
1767  Detector to get linearity type from.
1768 
1769  Returns
1770  -------
1771  doLinearize : `Bool`
1772  If True, linearization should be performed.
1773  """
1774  return self.config.doLinearize and \
1775  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1776 
1777  def flatCorrection(self, exposure, flatExposure, invert=False):
1778  """!Apply flat correction in place.
1779 
1780  Parameters
1781  ----------
1782  exposure : `lsst.afw.image.Exposure`
1783  Exposure to process.
1784  flatExposure : `lsst.afw.image.Exposure`
1785  Flat exposure of the same size as ``exposure``.
1786  invert : `Bool`, optional
1787  If True, unflatten an already flattened image.
1788 
1789  See Also
1790  --------
1791  lsst.ip.isr.isrFunctions.flatCorrection
1792  """
1793  isrFunctions.flatCorrection(
1794  maskedImage=exposure.getMaskedImage(),
1795  flatMaskedImage=flatExposure.getMaskedImage(),
1796  scalingType=self.config.flatScalingType,
1797  userScale=self.config.flatUserScale,
1798  invert=invert,
1799  trimToFit=self.config.doTrimToMatchCalib
1800  )
1801 
1802  def saturationDetection(self, exposure, amp):
1803  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1804 
1805  Parameters
1806  ----------
1807  exposure : `lsst.afw.image.Exposure`
1808  Exposure to process. Only the amplifier DataSec is processed.
1809  amp : `lsst.afw.table.AmpInfoCatalog`
1810  Amplifier detector data.
1811 
1812  See Also
1813  --------
1814  lsst.ip.isr.isrFunctions.makeThresholdMask
1815  """
1816  if not math.isnan(amp.getSaturation()):
1817  maskedImage = exposure.getMaskedImage()
1818  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1819  isrFunctions.makeThresholdMask(
1820  maskedImage=dataView,
1821  threshold=amp.getSaturation(),
1822  growFootprints=0,
1823  maskName=self.config.saturatedMaskName,
1824  )
1825 
1826  def saturationInterpolation(self, ccdExposure):
1827  """!Interpolate over saturated pixels, in place.
1828 
1829  This method should be called after `saturationDetection`, to
1830  ensure that the saturated pixels have been identified in the
1831  SAT mask. It should also be called after `assembleCcd`, since
1832  saturated regions may cross amplifier boundaries.
1833 
1834  Parameters
1835  ----------
1836  exposure : `lsst.afw.image.Exposure`
1837  Exposure to process.
1838 
1839  See Also
1840  --------
1841  lsst.ip.isr.isrTask.saturationDetection
1842  lsst.ip.isr.isrFunctions.interpolateFromMask
1843  """
1844  isrFunctions.interpolateFromMask(
1845  maskedImage=ccdExposure.getMaskedImage(),
1846  fwhm=self.config.fwhm,
1847  growFootprints=self.config.growSaturationFootprintSize,
1848  maskName=self.config.saturatedMaskName,
1849  )
1850 
1851  def suspectDetection(self, exposure, amp):
1852  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1853 
1854  Parameters
1855  ----------
1856  exposure : `lsst.afw.image.Exposure`
1857  Exposure to process. Only the amplifier DataSec is processed.
1858  amp : `lsst.afw.table.AmpInfoCatalog`
1859  Amplifier detector data.
1860 
1861  See Also
1862  --------
1863  lsst.ip.isr.isrFunctions.makeThresholdMask
1864 
1865  Notes
1866  -----
1867  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1868  This is intended to indicate pixels that may be affected by unknown systematics;
1869  for example if non-linearity corrections above a certain level are unstable
1870  then that would be a useful value for suspectLevel. A value of `nan` indicates
1871  that no such level exists and no pixels are to be masked as suspicious.
1872  """
1873  suspectLevel = amp.getSuspectLevel()
1874  if math.isnan(suspectLevel):
1875  return
1876 
1877  maskedImage = exposure.getMaskedImage()
1878  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1879  isrFunctions.makeThresholdMask(
1880  maskedImage=dataView,
1881  threshold=suspectLevel,
1882  growFootprints=0,
1883  maskName=self.config.suspectMaskName,
1884  )
1885 
1886  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1887  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1888 
1889  Parameters
1890  ----------
1891  ccdExposure : `lsst.afw.image.Exposure`
1892  Exposure to process.
1893  defectBaseList : `List`
1894  List of defects to mask and interpolate.
1895 
1896  Notes
1897  -----
1898  Call this after CCD assembly, since defects may cross amplifier boundaries.
1899  """
1900  maskedImage = ccdExposure.getMaskedImage()
1901  defectList = []
1902  for d in defectBaseList:
1903  bbox = d.getBBox()
1904  nd = measAlg.Defect(bbox)
1905  defectList.append(nd)
1906  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1907  isrFunctions.interpolateDefectList(
1908  maskedImage=maskedImage,
1909  defectList=defectList,
1910  fwhm=self.config.fwhm,
1911  )
1912 
1913  if self.config.numEdgeSuspect > 0:
1914  goodBBox = maskedImage.getBBox()
1915  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1916  goodBBox.grow(-self.config.numEdgeSuspect)
1917  # Mask pixels outside goodBBox as SUSPECT
1918  SourceDetectionTask.setEdgeBits(
1919  maskedImage,
1920  goodBBox,
1921  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1922  )
1923 
1924  def maskAndInterpNan(self, exposure):
1925  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1926 
1927  Parameters
1928  ----------
1929  exposure : `lsst.afw.image.Exposure`
1930  Exposure to process.
1931 
1932  Notes
1933  -----
1934  We mask and interpolate over all NaNs, including those
1935  that are masked with other bits (because those may or may
1936  not be interpolated over later, and we want to remove all
1937  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1938  is used to preserve the historical name.
1939  """
1940  maskedImage = exposure.getMaskedImage()
1941 
1942  # Find and mask NaNs
1943  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1944  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1945  numNans = maskNans(maskedImage, maskVal)
1946  self.metadata.set("NUMNANS", numNans)
1947 
1948  # Interpolate over these previously-unmasked NaNs
1949  if numNans > 0:
1950  self.log.warn("There were %i unmasked NaNs", numNans)
1951  nanDefectList = isrFunctions.getDefectListFromMask(
1952  maskedImage=maskedImage,
1953  maskName='UNMASKEDNAN',
1954  )
1955  isrFunctions.interpolateDefectList(
1956  maskedImage=exposure.getMaskedImage(),
1957  defectList=nanDefectList,
1958  fwhm=self.config.fwhm,
1959  )
1960 
1961  def measureBackground(self, exposure, IsrQaConfig=None):
1962  """Measure the image background in subgrids, for quality control purposes.
1963 
1964  Parameters
1965  ----------
1966  exposure : `lsst.afw.image.Exposure`
1967  Exposure to process.
1968  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
1969  Configuration object containing parameters on which background
1970  statistics and subgrids to use.
1971  """
1972  if IsrQaConfig is not None:
1973  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1974  IsrQaConfig.flatness.nIter)
1975  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
1976  statsControl.setAndMask(maskVal)
1977  maskedImage = exposure.getMaskedImage()
1978  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1979  skyLevel = stats.getValue(afwMath.MEDIAN)
1980  skySigma = stats.getValue(afwMath.STDEVCLIP)
1981  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1982  metadata = exposure.getMetadata()
1983  metadata.set('SKYLEVEL', skyLevel)
1984  metadata.set('SKYSIGMA', skySigma)
1985 
1986  # calcluating flatlevel over the subgrids
1987  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
1988  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1989  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
1990  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
1991  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
1992  skyLevels = numpy.zeros((nX, nY))
1993 
1994  for j in range(nY):
1995  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
1996  for i in range(nX):
1997  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
1998 
1999  xLLC = xc - meshXHalf
2000  yLLC = yc - meshYHalf
2001  xURC = xc + meshXHalf - 1
2002  yURC = yc + meshYHalf - 1
2003 
2004  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2005  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2006 
2007  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2008 
2009  good = numpy.where(numpy.isfinite(skyLevels))
2010  skyMedian = numpy.median(skyLevels[good])
2011  flatness = (skyLevels[good] - skyMedian) / skyMedian
2012  flatness_rms = numpy.std(flatness)
2013  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2014 
2015  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2016  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2017  (nX, nY, flatness_pp, flatness_rms))
2018 
2019  metadata.set('FLATNESS_PP', float(flatness_pp))
2020  metadata.set('FLATNESS_RMS', float(flatness_rms))
2021  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2022  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2023  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2024 
2025  def roughZeroPoint(self, exposure):
2026  """Set an approximate magnitude zero point for the exposure.
2027 
2028  Parameters
2029  ----------
2030  exposure : `lsst.afw.image.Exposure`
2031  Exposure to process.
2032  """
2033  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2034  if filterName in self.config.fluxMag0T1:
2035  fluxMag0 = self.config.fluxMag0T1[filterName]
2036  else:
2037  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2038  fluxMag0 = self.config.defaultFluxMag0T1
2039 
2040  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2041  if not expTime > 0: # handle NaN as well as <= 0
2042  self.log.warn("Non-positive exposure time; skipping rough zero point")
2043  return
2044 
2045  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2046  exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2047 
2048  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2049  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2050 
2051  Parameters
2052  ----------
2053  ccdExposure : `lsst.afw.image.Exposure`
2054  Exposure to process.
2055  fpPolygon : `lsst.afw.geom.Polygon`
2056  Polygon in focal plane coordinates.
2057  """
2058  # Get ccd corners in focal plane coordinates
2059  ccd = ccdExposure.getDetector()
2060  fpCorners = ccd.getCorners(FOCAL_PLANE)
2061  ccdPolygon = Polygon(fpCorners)
2062 
2063  # Get intersection of ccd corners with fpPolygon
2064  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2065 
2066  # Transform back to pixel positions and build new polygon
2067  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2068  validPolygon = Polygon(ccdPoints)
2069  ccdExposure.getInfo().setValidPolygon(validPolygon)
2070 
2071  @contextmanager
2072  def flatContext(self, exp, flat, dark=None):
2073  """Context manager that applies and removes flats and darks,
2074  if the task is configured to apply them.
2075 
2076  Parameters
2077  ----------
2078  exp : `lsst.afw.image.Exposure`
2079  Exposure to process.
2080  flat : `lsst.afw.image.Exposure`
2081  Flat exposure the same size as ``exp``.
2082  dark : `lsst.afw.image.Exposure`, optional
2083  Dark exposure the same size as ``exp``.
2084 
2085  Yields
2086  ------
2087  exp : `lsst.afw.image.Exposure`
2088  The flat and dark corrected exposure.
2089  """
2090  if self.config.doDark and dark is not None:
2091  self.darkCorrection(exp, dark)
2092  if self.config.doFlat:
2093  self.flatCorrection(exp, flat)
2094  try:
2095  yield exp
2096  finally:
2097  if self.config.doFlat:
2098  self.flatCorrection(exp, flat, invert=True)
2099  if self.config.doDark and dark is not None:
2100  self.darkCorrection(exp, dark, invert=True)
2101 
2102  def debugView(self, exposure, stepname):
2103  """Utility function to examine ISR exposure at different stages.
2104 
2105  Parameters
2106  ----------
2107  exposure : `lsst.afw.image.Exposure`
2108  Exposure to view.
2109  stepname : `str`
2110  State of processing to view.
2111  """
2112  frame = getDebugFrame(self._display, stepname)
2113  if frame:
2114  display = getDisplay(frame)
2115  display.scale('asinh', 'zscale')
2116  display.mtv(exposure)
2117 
2118 
2119 class FakeAmp(object):
2120  """A Detector-like object that supports returning gain and saturation level
2121 
2122  This is used when the input exposure does not have a detector.
2123 
2124  Parameters
2125  ----------
2126  exposure : `lsst.afw.image.Exposure`
2127  Exposure to generate a fake amplifier for.
2128  config : `lsst.ip.isr.isrTaskConfig`
2129  Configuration to apply to the fake amplifier.
2130  """
2131 
2132  def __init__(self, exposure, config):
2133  self._bbox = exposure.getBBox(afwImage.LOCAL)
2134  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
2135  self._gain = config.gain
2136  self._readNoise = config.readNoise
2137  self._saturation = config.saturation
2138 
2139  def getBBox(self):
2140  return self._bbox
2141 
2142  def getRawBBox(self):
2143  return self._bbox
2144 
2145  def getHasRawInfo(self):
2146  return True # but see getRawHorizontalOverscanBBox()
2147 
2149  return self._RawHorizontalOverscanBBox
2150 
2151  def getGain(self):
2152  return self._gain
2153 
2154  def getReadNoise(self):
2155  return self._readNoise
2156 
2157  def getSaturation(self):
2158  return self._saturation
2159 
2160  def getSuspectLevel(self):
2161  return float("NaN")
2162 
2163 
2164 class RunIsrConfig(pexConfig.Config):
2165  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2166 
2167 
2168 class RunIsrTask(pipeBase.CmdLineTask):
2169  """Task to wrap the default IsrTask to allow it to be retargeted.
2170 
2171  The standard IsrTask can be called directly from a command line
2172  program, but doing so removes the ability of the task to be
2173  retargeted. As most cameras override some set of the IsrTask
2174  methods, this would remove those data-specific methods in the
2175  output post-ISR images. This wrapping class fixes the issue,
2176  allowing identical post-ISR images to be generated by both the
2177  processCcd and isrTask code.
2178  """
2179  ConfigClass = RunIsrConfig
2180  _DefaultName = "runIsr"
2181 
2182  def __init__(self, *args, **kwargs):
2183  super().__init__(*args, **kwargs)
2184  self.makeSubtask("isr")
2185 
2186  def runDataRef(self, dataRef):
2187  """
2188  Parameters
2189  ----------
2190  dataRef : `lsst.daf.persistence.ButlerDataRef`
2191  data reference of the detector data to be processed
2192 
2193  Returns
2194  -------
2195  result : `pipeBase.Struct`
2196  Result struct with component:
2197 
2198  - exposure : `lsst.afw.image.Exposure`
2199  Post-ISR processed exposure.
2200  """
2201  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:706
def runDataRef(self, sensorRef)
Definition: isrTask.py:1287
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:1961
def debugView(self, exposure, stepname)
Definition: isrTask.py:2102
def __init__(self, kwargs)
Definition: isrTask.py:696
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1380
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:801
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:755
def runDataRef(self, dataRef)
Definition: isrTask.py:2186
def __init__(self, args, kwargs)
Definition: isrTask.py:2182
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1924
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1826
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2025
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2148
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:743
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1528
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1424
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1777
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:798
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1339
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1715
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1758
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:905
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2048
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1460
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2072
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1666
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1851
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1886
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1802
def __init__(self, exposure, config)
Definition: isrTask.py:2132