lsst.ip.isr  17.0.1-10-g9695de4
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defect
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 __all__ = ["IsrTask", "RunIsrTask"]
57 
58 
59 class IsrTaskConfig(pexConfig.Config):
60  """Configuration parameters for IsrTask.
61 
62  Items are grouped in the order in which they are executed by the task.
63  """
64  # General ISR configuration
65 
66  # gen3 options
67  isrName = pexConfig.Field(
68  dtype=str,
69  doc="Name of ISR",
70  default="ISR",
71  )
72 
73  # input datasets
74  ccdExposure = pipeBase.InputDatasetField(
75  doc="Input exposure to process",
76  name="raw",
77  scalar=True,
78  storageClass="ExposureU",
79  dimensions=["Instrument", "Exposure", "Detector"],
80  )
81  camera = pipeBase.InputDatasetField(
82  doc="Input camera to construct complete exposures.",
83  name="camera",
84  scalar=True,
85  storageClass="TablePersistableCamera",
86  dimensions=["Instrument", "CalibrationLabel"],
87  )
88  bias = pipeBase.InputDatasetField(
89  doc="Input bias calibration.",
90  name="bias",
91  scalar=True,
92  storageClass="ImageF",
93  dimensions=["Instrument", "CalibrationLabel", "Detector"],
94  )
95  dark = pipeBase.InputDatasetField(
96  doc="Input dark calibration.",
97  name="dark",
98  scalar=True,
99  storageClass="ImageF",
100  dimensions=["Instrument", "CalibrationLabel", "Detector"],
101  )
102  flat = pipeBase.InputDatasetField(
103  doc="Input flat calibration.",
104  name="flat",
105  scalar=True,
106  storageClass="MaskedImageF",
107  dimensions=["Instrument", "PhysicalFilter", "CalibrationLabel", "Detector"],
108  )
109  bfKernel = pipeBase.InputDatasetField(
110  doc="Input brighter-fatter kernel.",
111  name="bfKernel",
112  scalar=True,
113  storageClass="NumpyArray",
114  dimensions=["Instrument", "CalibrationLabel"],
115  )
116  defects = pipeBase.InputDatasetField(
117  doc="Input defect tables.",
118  name="defects",
119  scalar=True,
120  storageClass="Catalog",
121  dimensions=["Instrument", "CalibrationLabel", "Detector"],
122  )
123  opticsTransmission = pipeBase.InputDatasetField(
124  doc="Transmission curve due to the optics.",
125  name="transmission_optics",
126  scalar=True,
127  storageClass="TablePersistableTransmissionCurve",
128  dimensions=["Instrument", "CalibrationLabel"],
129  )
130  filterTransmission = pipeBase.InputDatasetField(
131  doc="Transmission curve due to the filter.",
132  name="transmission_filter",
133  scalar=True,
134  storageClass="TablePersistableTransmissionCurve",
135  dimensions=["Instrument", "PhysicalFilter", "CalibrationLabel"],
136  )
137  sensorTransmission = pipeBase.InputDatasetField(
138  doc="Transmission curve due to the sensor.",
139  name="transmission_sensor",
140  scalar=True,
141  storageClass="TablePersistableTransmissionCurve",
142  dimensions=["Instrument", "CalibrationLabel", "Detector"],
143  )
144  atmosphereTransmission = pipeBase.InputDatasetField(
145  doc="Transmission curve due to the atmosphere.",
146  name="transmission_atmosphere",
147  scalar=True,
148  storageClass="TablePersistableTransmissionCurve",
149  dimensions=["Instrument"],
150  )
151 
152  # output datasets
153  outputExposure = pipeBase.OutputDatasetField(
154  doc="Output ISR processed exposure.",
155  name="postISRCCD",
156  scalar=True,
157  storageClass="ExposureF",
158  dimensions=["Instrument", "Visit", "Detector"],
159  )
160  outputOssThumbnail = pipeBase.OutputDatasetField(
161  doc="Output Overscan-subtracted thumbnail image.",
162  name="OssThumb",
163  scalar=True,
164  storageClass="Thumbnail",
165  dimensions=["Instrument", "Visit", "Detector"],
166  )
167  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
168  doc="Output flat-corrected thumbnail image.",
169  name="FlattenedThumb",
170  scalar=True,
171  storageClass="TextStorage",
172  dimensions=["Instrument", "Visit", "Detector"],
173  )
174 
175  quantum = pipeBase.QuantumConfig(
176  dimensions=["Visit", "Detector", "Instrument"],
177  )
178 
179 
180  datasetType = pexConfig.Field(
181  dtype=str,
182  doc="Dataset type for input data; users will typically leave this alone, "
183  "but camera-specific ISR tasks will override it",
184  default="raw",
185  )
186 
187  fallbackFilterName = pexConfig.Field(
188  dtype=str,
189  doc="Fallback default filter name for calibrations.",
190  optional=True
191  )
192  expectWcs = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
196  )
197  fwhm = pexConfig.Field(
198  dtype=float,
199  doc="FWHM of PSF in arcseconds.",
200  default=1.0,
201  )
202  qa = pexConfig.ConfigField(
203  dtype=isrQa.IsrQaConfig,
204  doc="QA related configuration options.",
205  )
206 
207  # Image conversion configuration
208  doConvertIntToFloat = pexConfig.Field(
209  dtype=bool,
210  doc="Convert integer raw images to floating point values?",
211  default=True,
212  )
213 
214  # Saturated pixel handling.
215  doSaturation = pexConfig.Field(
216  dtype=bool,
217  doc="Mask saturated pixels? NB: this is totally independent of the"
218  " interpolation option - this is ONLY setting the bits in the mask."
219  " To have them interpolated make sure doSaturationInterpolation=True",
220  default=True,
221  )
222  saturatedMaskName = pexConfig.Field(
223  dtype=str,
224  doc="Name of mask plane to use in saturation detection and interpolation",
225  default="SAT",
226  )
227  saturation = pexConfig.Field(
228  dtype=float,
229  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
230  default=float("NaN"),
231  )
232  growSaturationFootprintSize = pexConfig.Field(
233  dtype=int,
234  doc="Number of pixels by which to grow the saturation footprints",
235  default=1,
236  )
237 
238  # Suspect pixel handling.
239  doSuspect = pexConfig.Field(
240  dtype=bool,
241  doc="Mask suspect pixels?",
242  default=True,
243  )
244  suspectMaskName = pexConfig.Field(
245  dtype=str,
246  doc="Name of mask plane to use for suspect pixels",
247  default="SUSPECT",
248  )
249  numEdgeSuspect = pexConfig.Field(
250  dtype=int,
251  doc="Number of edge pixels to be flagged as untrustworthy.",
252  default=0,
253  )
254 
255  # Initial masking options.
256  doSetBadRegions = pexConfig.Field(
257  dtype=bool,
258  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
259  default=True,
260  )
261  badStatistic = pexConfig.ChoiceField(
262  dtype=str,
263  doc="How to estimate the average value for BAD regions.",
264  default='MEANCLIP',
265  allowed={
266  "MEANCLIP": "Correct using the (clipped) mean of good data",
267  "MEDIAN": "Correct using the median of the good data",
268  },
269  )
270 
271  # Overscan subtraction configuration.
272  doOverscan = pexConfig.Field(
273  dtype=bool,
274  doc="Do overscan subtraction?",
275  default=True,
276  )
277  overscanFitType = pexConfig.ChoiceField(
278  dtype=str,
279  doc="The method for fitting the overscan bias level.",
280  default='MEDIAN',
281  allowed={
282  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
283  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
284  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
285  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
286  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
287  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
288  "MEAN": "Correct using the mean of the overscan region",
289  "MEANCLIP": "Correct using a clipped mean of the overscan region",
290  "MEDIAN": "Correct using the median of the overscan region",
291  },
292  )
293  overscanOrder = pexConfig.Field(
294  dtype=int,
295  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
296  "or number of spline knots if overscan fit type is a spline."),
297  default=1,
298  )
299  overscanNumSigmaClip = pexConfig.Field(
300  dtype=float,
301  doc="Rejection threshold (sigma) for collapsing overscan before fit",
302  default=3.0,
303  )
304  overscanIsInt = pexConfig.Field(
305  dtype=bool,
306  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
307  default=True,
308  )
309  overscanNumLeadingColumnsToSkip = pexConfig.Field(
310  dtype=int,
311  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
312  default=0,
313  )
314  overscanNumTrailingColumnsToSkip = pexConfig.Field(
315  dtype=int,
316  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
317  default=0,
318  )
319  overscanMaxDev = pexConfig.Field(
320  dtype=float,
321  doc="Maximum deviation from the median for overscan",
322  default=1000.0, check=lambda x: x > 0
323  )
324  overscanBiasJump = pexConfig.Field(
325  dtype=bool,
326  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
327  default=False,
328  )
329  overscanBiasJumpKeyword = pexConfig.Field(
330  dtype=str,
331  doc="Header keyword containing information about devices.",
332  default="NO_SUCH_KEY",
333  )
334  overscanBiasJumpDevices = pexConfig.ListField(
335  dtype=str,
336  doc="List of devices that need piecewise overscan correction.",
337  default=(),
338  )
339  overscanBiasJumpLocation = pexConfig.Field(
340  dtype=int,
341  doc="Location of bias jump along y-axis.",
342  default=0,
343  )
344 
345  # Amplifier to CCD assembly configuration
346  doAssembleCcd = pexConfig.Field(
347  dtype=bool,
348  default=True,
349  doc="Assemble amp-level exposures into a ccd-level exposure?"
350  )
351  assembleCcd = pexConfig.ConfigurableField(
352  target=AssembleCcdTask,
353  doc="CCD assembly task",
354  )
355 
356  # General calibration configuration.
357  doAssembleIsrExposures = pexConfig.Field(
358  dtype=bool,
359  default=False,
360  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
361  )
362  doTrimToMatchCalib = pexConfig.Field(
363  dtype=bool,
364  default=False,
365  doc="Trim raw data to match calibration bounding boxes?"
366  )
367 
368  # Bias subtraction.
369  doBias = pexConfig.Field(
370  dtype=bool,
371  doc="Apply bias frame correction?",
372  default=True,
373  )
374  biasDataProductName = pexConfig.Field(
375  dtype=str,
376  doc="Name of the bias data product",
377  default="bias",
378  )
379 
380  # Variance construction
381  doVariance = pexConfig.Field(
382  dtype=bool,
383  doc="Calculate variance?",
384  default=True
385  )
386  gain = pexConfig.Field(
387  dtype=float,
388  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
389  default=float("NaN"),
390  )
391  readNoise = pexConfig.Field(
392  dtype=float,
393  doc="The read noise to use if no Detector is present in the Exposure",
394  default=0.0,
395  )
396  doEmpiricalReadNoise = pexConfig.Field(
397  dtype=bool,
398  default=False,
399  doc="Calculate empirical read noise instead of value from AmpInfo data?"
400  )
401 
402  # Linearization.
403  doLinearize = pexConfig.Field(
404  dtype=bool,
405  doc="Correct for nonlinearity of the detector's response?",
406  default=True,
407  )
408 
409  # Crosstalk.
410  doCrosstalk = pexConfig.Field(
411  dtype=bool,
412  doc="Apply intra-CCD crosstalk correction?",
413  default=False,
414  )
415  doCrosstalkBeforeAssemble = pexConfig.Field(
416  dtype=bool,
417  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
418  default=True,
419  )
420  crosstalk = pexConfig.ConfigurableField(
421  target=CrosstalkTask,
422  doc="Intra-CCD crosstalk correction",
423  )
424 
425  # Masking option prior to brighter-fatter.
426  doWidenSaturationTrails = pexConfig.Field(
427  dtype=bool,
428  doc="Widen bleed trails based on their width?",
429  default=True
430  )
431 
432  # Brighter-Fatter correction.
433  doBrighterFatter = pexConfig.Field(
434  dtype=bool,
435  default=False,
436  doc="Apply the brighter fatter correction"
437  )
438  brighterFatterLevel = pexConfig.ChoiceField(
439  dtype=str,
440  default="DETECTOR",
441  doc="The level at which to correct for brighter-fatter.",
442  allowed={
443  "AMP": "Every amplifier treated separately.",
444  "DETECTOR": "One kernel per detector",
445  }
446  )
447  brighterFatterKernelFile = pexConfig.Field(
448  dtype=str,
449  default='',
450  doc="Kernel file used for the brighter fatter correction"
451  )
452  brighterFatterMaxIter = pexConfig.Field(
453  dtype=int,
454  default=10,
455  doc="Maximum number of iterations for the brighter fatter correction"
456  )
457  brighterFatterThreshold = pexConfig.Field(
458  dtype=float,
459  default=1000,
460  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
461  " absolute value of the difference between the current corrected image and the one"
462  " from the previous iteration summed over all the pixels."
463  )
464  brighterFatterApplyGain = pexConfig.Field(
465  dtype=bool,
466  default=True,
467  doc="Should the gain be applied when applying the brighter fatter correction?"
468  )
469 
470  # Defect and bad pixel correction options.
471  doDefect = pexConfig.Field(
472  dtype=bool,
473  doc="Apply correction for CCD defects, e.g. hot pixels?",
474  default=True,
475  )
476  doSaturationInterpolation = pexConfig.Field(
477  dtype=bool,
478  doc="Perform interpolation over pixels masked as saturated?"
479  " NB: This is independent of doSaturation; if that is False this plane"
480  " will likely be blank, resulting in a no-op here.",
481  default=True,
482  )
483  numEdgeSuspect = pexConfig.Field(
484  dtype=int,
485  doc="Number of edge pixels to be flagged as untrustworthy.",
486  default=0,
487  )
488 
489  # Dark subtraction.
490  doDark = pexConfig.Field(
491  dtype=bool,
492  doc="Apply dark frame correction?",
493  default=True,
494  )
495  darkDataProductName = pexConfig.Field(
496  dtype=str,
497  doc="Name of the dark data product",
498  default="dark",
499  )
500 
501  # Camera-specific stray light removal.
502  doStrayLight = pexConfig.Field(
503  dtype=bool,
504  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
505  default=False,
506  )
507  strayLight = pexConfig.ConfigurableField(
508  target=StrayLightTask,
509  doc="y-band stray light correction"
510  )
511 
512  # Flat correction.
513  doFlat = pexConfig.Field(
514  dtype=bool,
515  doc="Apply flat field correction?",
516  default=True,
517  )
518  flatDataProductName = pexConfig.Field(
519  dtype=str,
520  doc="Name of the flat data product",
521  default="flat",
522  )
523  flatScalingType = pexConfig.ChoiceField(
524  dtype=str,
525  doc="The method for scaling the flat on the fly.",
526  default='USER',
527  allowed={
528  "USER": "Scale by flatUserScale",
529  "MEAN": "Scale by the inverse of the mean",
530  "MEDIAN": "Scale by the inverse of the median",
531  },
532  )
533  flatUserScale = pexConfig.Field(
534  dtype=float,
535  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
536  default=1.0,
537  )
538  doTweakFlat = pexConfig.Field(
539  dtype=bool,
540  doc="Tweak flats to match observed amplifier ratios?",
541  default=False
542  )
543 
544  # Amplifier normalization based on gains instead of using flats configuration.
545  doApplyGains = pexConfig.Field(
546  dtype=bool,
547  doc="Correct the amplifiers for their gains instead of applying flat correction",
548  default=False,
549  )
550  normalizeGains = pexConfig.Field(
551  dtype=bool,
552  doc="Normalize all the amplifiers in each CCD to have the same median value.",
553  default=False,
554  )
555 
556  # Fringe correction.
557  doFringe = pexConfig.Field(
558  dtype=bool,
559  doc="Apply fringe correction?",
560  default=True,
561  )
562  fringe = pexConfig.ConfigurableField(
563  target=FringeTask,
564  doc="Fringe subtraction task",
565  )
566  fringeAfterFlat = pexConfig.Field(
567  dtype=bool,
568  doc="Do fringe subtraction after flat-fielding?",
569  default=True,
570  )
571 
572  # NAN pixel interpolation option.
573  doNanInterpAfterFlat = pexConfig.Field(
574  dtype=bool,
575  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
576  "also have to interpolate them before flat-fielding."),
577  default=False,
578  )
579 
580  # Distortion model application.
581  doAddDistortionModel = pexConfig.Field(
582  dtype=bool,
583  doc="Apply a distortion model based on camera geometry to the WCS?",
584  default=True,
585  )
586 
587  # Initial CCD-level background statistics options.
588  doMeasureBackground = pexConfig.Field(
589  dtype=bool,
590  doc="Measure the background level on the reduced image?",
591  default=False,
592  )
593 
594  # Camera-specific masking configuration.
595  doCameraSpecificMasking = pexConfig.Field(
596  dtype=bool,
597  doc="Mask camera-specific bad regions?",
598  default=False,
599  )
600  masking = pexConfig.ConfigurableField(
601  target=MaskingTask,
602  doc="Masking task."
603  )
604 
605  # Default photometric calibration options.
606  fluxMag0T1 = pexConfig.DictField(
607  keytype=str,
608  itemtype=float,
609  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
610  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
611  ))
612  )
613  defaultFluxMag0T1 = pexConfig.Field(
614  dtype=float,
615  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
616  default=pow(10.0, 0.4*28.0)
617  )
618 
619  # Vignette correction configuration.
620  doVignette = pexConfig.Field(
621  dtype=bool,
622  doc="Apply vignetting parameters?",
623  default=False,
624  )
625  vignette = pexConfig.ConfigurableField(
626  target=VignetteTask,
627  doc="Vignetting task.",
628  )
629 
630  # Transmission curve configuration.
631  doAttachTransmissionCurve = pexConfig.Field(
632  dtype=bool,
633  default=False,
634  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
635  )
636  doUseOpticsTransmission = pexConfig.Field(
637  dtype=bool,
638  default=True,
639  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
640  )
641  doUseFilterTransmission = pexConfig.Field(
642  dtype=bool,
643  default=True,
644  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
645  )
646  doUseSensorTransmission = pexConfig.Field(
647  dtype=bool,
648  default=True,
649  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
650  )
651  doUseAtmosphereTransmission = pexConfig.Field(
652  dtype=bool,
653  default=True,
654  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
655  )
656 
657  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
658  doWrite = pexConfig.Field(
659  dtype=bool,
660  doc="Persist postISRCCD?",
661  default=True,
662  )
663 
664  def validate(self):
665  super().validate()
666  if self.doFlat and self.doApplyGains:
667  raise ValueError("You may not specify both doFlat and doApplyGains")
668 
669 
670 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
671  r"""Apply common instrument signature correction algorithms to a raw frame.
672 
673  The process for correcting imaging data is very similar from
674  camera to camera. This task provides a vanilla implementation of
675  doing these corrections, including the ability to turn certain
676  corrections off if they are not needed. The inputs to the primary
677  method, `run()`, are a raw exposure to be corrected and the
678  calibration data products. The raw input is a single chip sized
679  mosaic of all amps including overscans and other non-science
680  pixels. The method `runDataRef()` identifies and defines the
681  calibration data products, and is intended for use by a
682  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
683  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
684  subclassed for different camera, although the most camera specific
685  methods have been split into subtasks that can be redirected
686  appropriately.
687 
688  The __init__ method sets up the subtasks for ISR processing, using
689  the defaults from `lsst.ip.isr`.
690 
691  Parameters
692  ----------
693  args : `list`
694  Positional arguments passed to the Task constructor. None used at this time.
695  kwargs : `dict`, optional
696  Keyword arguments passed on to the Task constructor. None used at this time.
697  """
698  ConfigClass = IsrTaskConfig
699  _DefaultName = "isr"
700 
701  def __init__(self, **kwargs):
702  super().__init__(**kwargs)
703  self.makeSubtask("assembleCcd")
704  self.makeSubtask("crosstalk")
705  self.makeSubtask("strayLight")
706  self.makeSubtask("fringe")
707  self.makeSubtask("masking")
708  self.makeSubtask("vignette")
709 
710  @classmethod
711  def getInputDatasetTypes(cls, config):
712  inputTypeDict = super().getInputDatasetTypes(config)
713 
714  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
715  # need because the configuration tells us we will not be bothering with the
716  # correction that uses that IDT.
717  if config.doBias is not True:
718  inputTypeDict.pop("bias", None)
719  if config.doLinearize is not True:
720  inputTypeDict.pop("linearizer", None)
721  if config.doCrosstalk is not True:
722  inputTypeDict.pop("crosstalkSources", None)
723  if config.doBrighterFatter is not True:
724  inputTypeDict.pop("bfKernel", None)
725  if config.doDefect is not True:
726  inputTypeDict.pop("defects", None)
727  if config.doDark is not True:
728  inputTypeDict.pop("dark", None)
729  if config.doFlat is not True:
730  inputTypeDict.pop("flat", None)
731  if config.doAttachTransmissionCurve is not True:
732  inputTypeDict.pop("opticsTransmission", None)
733  inputTypeDict.pop("filterTransmission", None)
734  inputTypeDict.pop("sensorTransmission", None)
735  inputTypeDict.pop("atmosphereTransmission", None)
736  if config.doUseOpticsTransmission is not True:
737  inputTypeDict.pop("opticsTransmission", None)
738  if config.doUseFilterTransmission is not True:
739  inputTypeDict.pop("filterTransmission", None)
740  if config.doUseSensorTransmission is not True:
741  inputTypeDict.pop("sensorTransmission", None)
742  if config.doUseAtmosphereTransmission is not True:
743  inputTypeDict.pop("atmosphereTransmission", None)
744 
745  return inputTypeDict
746 
747  @classmethod
748  def getOutputDatasetTypes(cls, config):
749  outputTypeDict = super().getOutputDatasetTypes(config)
750 
751  if config.qa.doThumbnailOss is not True:
752  outputTypeDict.pop("outputOssThumbnail", None)
753  if config.qa.doThumbnailFlattened is not True:
754  outputTypeDict.pop("outputFlattenedThumbnail", None)
755  if config.doWrite is not True:
756  outputTypeDict.pop("outputExposure", None)
757 
758  return outputTypeDict
759 
760  @classmethod
761  def getPrerequisiteDatasetTypes(cls, config):
762  # Input calibration datasets should not constrain the QuantumGraph
763  # (it'd be confusing if not having flats just silently resulted in no
764  # data being processed). Our nomenclature for that is that these are
765  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
766  names = set(cls.getInputDatasetTypes(config))
767  names.remove("ccdExposure")
768  return names
769 
770  @classmethod
771  def getPerDatasetTypeDimensions(cls, config):
772  # Input calibration datasets of different types (i.e. flat, bias) need
773  # not have the same validity range. That makes CalibrationLabel
774  # (which maps directly to a validity range) a "per-DatasetType
775  # dimension".
776  return frozenset(["CalibrationLabel"])
777 
778  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
779  try:
780  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
781  except Exception as e:
782  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
783 
784  inputData['isGen3'] = True
785 
786  if self.config.doLinearize is True:
787  if 'linearizer' not in inputData.keys():
788  detector = inputData['camera'][inputData['detectorNum']]
789  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
790  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
791 
792  if inputData['defects'] is not None:
793  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
794  # masking expects a list of defects defined by their bounding box
795  defectList = []
796 
797  for r in inputData['defects']:
798  bbox = afwGeom.BoxI(afwGeom.PointI(r.get("x0"), r.get("y0")),
799  afwGeom.ExtentI(r.get("width"), r.get("height")))
800  defectList.append(Defect(bbox))
801 
802  inputData['defects'] = defectList
803 
804  # Broken: DM-17169
805  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
806  # fixed for non-HSC cameras in the future.
807  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
808  # if self.config.doCrosstalk else None)
809 
810  # Broken: DM-17152
811  # Fringes are not tested to be handled correctly by Gen3 butler.
812  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
813  # assembler=self.assembleCcd
814  # if self.config.doAssembleIsrExposures else None)
815  # if self.config.doFringe and
816  # self.fringe.checkFilter(inputData['ccdExposure'])
817  # else pipeBase.Struct(fringes=None))
818 
819  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
820 
821  def makeDatasetType(self, dsConfig):
822  return super().makeDatasetType(dsConfig)
823 
824  def readIsrData(self, dataRef, rawExposure):
825  """!Retrieve necessary frames for instrument signature removal.
826 
827  Pre-fetching all required ISR data products limits the IO
828  required by the ISR. Any conflict between the calibration data
829  available and that needed for ISR is also detected prior to
830  doing processing, allowing it to fail quickly.
831 
832  Parameters
833  ----------
834  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
835  Butler reference of the detector data to be processed
836  rawExposure : `afw.image.Exposure`
837  The raw exposure that will later be corrected with the
838  retrieved calibration data; should not be modified in this
839  method.
840 
841  Returns
842  -------
843  result : `lsst.pipe.base.Struct`
844  Result struct with components (which may be `None`):
845  - ``bias``: bias calibration frame (`afw.image.Exposure`)
846  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
847  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
848  - ``dark``: dark calibration frame (`afw.image.Exposure`)
849  - ``flat``: flat calibration frame (`afw.image.Exposure`)
850  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
851  - ``defects``: list of defects (`list`)
852  - ``fringes``: `lsst.pipe.base.Struct` with components:
853  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
854  - ``seed``: random seed derived from the ccdExposureId for random
855  number generator (`uint32`)
856  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
857  A ``TransmissionCurve`` that represents the throughput of the optics,
858  to be evaluated in focal-plane coordinates.
859  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
860  A ``TransmissionCurve`` that represents the throughput of the filter
861  itself, to be evaluated in focal-plane coordinates.
862  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
863  A ``TransmissionCurve`` that represents the throughput of the sensor
864  itself, to be evaluated in post-assembly trimmed detector coordinates.
865  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
866  A ``TransmissionCurve`` that represents the throughput of the
867  atmosphere, assumed to be spatially constant.
868  - ``strayLightData`` : `object`
869  An opaque object containing calibration information for
870  stray-light correction. If `None`, no correction will be
871  performed.
872 
873  Raises
874  ------
875  NotImplementedError :
876  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
877  """
878  ccd = rawExposure.getDetector()
879  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
880  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
881  if self.config.doBias else None)
882  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
883  linearizer = (dataRef.get("linearizer", immediate=True)
884  if self.doLinearize(ccd) else None)
885  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
886  if self.config.doCrosstalk else None)
887  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
888  if self.config.doDark else None)
889  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
890  if self.config.doFlat else None)
891 
892  brighterFatterKernel = None
893  if self.config.doBrighterFatter is True:
894 
895  # Use the new-style cp_pipe version of the kernel is it exists.
896  try:
897  brighterFatterKernel = dataRef.get("brighterFatterKernel")
898  except NoResults:
899  # Fall back to the old-style numpy-ndarray style kernel if necessary.
900  try:
901  brighterFatterKernel = dataRef.get("bfKernel")
902  except NoResults:
903  brighterFatterKernel = None
904  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
905  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
906  # this detector, or raise an error.
907  if self.config.brighterFatterLevel == 'DETECTOR':
908  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
909  else:
910  # TODO DM-15631 for implementing this
911  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
912 
913  defectList = (dataRef.get("defects")
914  if self.config.doDefect else None)
915  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
916  if self.config.doAssembleIsrExposures else None)
917  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
918  else pipeBase.Struct(fringes=None))
919 
920  if self.config.doAttachTransmissionCurve:
921  opticsTransmission = (dataRef.get("transmission_optics")
922  if self.config.doUseOpticsTransmission else None)
923  filterTransmission = (dataRef.get("transmission_filter")
924  if self.config.doUseFilterTransmission else None)
925  sensorTransmission = (dataRef.get("transmission_sensor")
926  if self.config.doUseSensorTransmission else None)
927  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
928  if self.config.doUseAtmosphereTransmission else None)
929  else:
930  opticsTransmission = None
931  filterTransmission = None
932  sensorTransmission = None
933  atmosphereTransmission = None
934 
935  if self.config.doStrayLight:
936  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
937  else:
938  strayLightData = None
939 
940  # Struct should include only kwargs to run()
941  return pipeBase.Struct(bias=biasExposure,
942  linearizer=linearizer,
943  crosstalkSources=crosstalkSources,
944  dark=darkExposure,
945  flat=flatExposure,
946  bfKernel=brighterFatterKernel,
947  defects=defectList,
948  fringes=fringeStruct,
949  opticsTransmission=opticsTransmission,
950  filterTransmission=filterTransmission,
951  sensorTransmission=sensorTransmission,
952  atmosphereTransmission=atmosphereTransmission,
953  strayLightData=strayLightData
954  )
955 
956  @pipeBase.timeMethod
957  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
958  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
959  opticsTransmission=None, filterTransmission=None,
960  sensorTransmission=None, atmosphereTransmission=None,
961  detectorNum=None, strayLightData=None, isGen3=False,
962  ):
963  """!Perform instrument signature removal on an exposure.
964 
965  Steps included in the ISR processing, in order performed, are:
966  - saturation and suspect pixel masking
967  - overscan subtraction
968  - CCD assembly of individual amplifiers
969  - bias subtraction
970  - variance image construction
971  - linearization of non-linear response
972  - crosstalk masking
973  - brighter-fatter correction
974  - dark subtraction
975  - fringe correction
976  - stray light subtraction
977  - flat correction
978  - masking of known defects and camera specific features
979  - vignette calculation
980  - appending transmission curve and distortion model
981 
982  Parameters
983  ----------
984  ccdExposure : `lsst.afw.image.Exposure`
985  The raw exposure that is to be run through ISR. The
986  exposure is modified by this method.
987  camera : `lsst.afw.cameraGeom.Camera`, optional
988  The camera geometry for this exposure. Used to select the
989  distortion model appropriate for this data.
990  bias : `lsst.afw.image.Exposure`, optional
991  Bias calibration frame.
992  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
993  Functor for linearization.
994  crosstalkSources : `list`, optional
995  List of possible crosstalk sources.
996  dark : `lsst.afw.image.Exposure`, optional
997  Dark calibration frame.
998  flat : `lsst.afw.image.Exposure`, optional
999  Flat calibration frame.
1000  bfKernel : `numpy.ndarray`, optional
1001  Brighter-fatter kernel.
1002  defects : `list`, optional
1003  List of defects.
1004  fringes : `lsst.pipe.base.Struct`, optional
1005  Struct containing the fringe correction data, with
1006  elements:
1007  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1008  - ``seed``: random seed derived from the ccdExposureId for random
1009  number generator (`uint32`)
1010  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1011  A ``TransmissionCurve`` that represents the throughput of the optics,
1012  to be evaluated in focal-plane coordinates.
1013  filterTransmission : `lsst.afw.image.TransmissionCurve`
1014  A ``TransmissionCurve`` that represents the throughput of the filter
1015  itself, to be evaluated in focal-plane coordinates.
1016  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1017  A ``TransmissionCurve`` that represents the throughput of the sensor
1018  itself, to be evaluated in post-assembly trimmed detector coordinates.
1019  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1020  A ``TransmissionCurve`` that represents the throughput of the
1021  atmosphere, assumed to be spatially constant.
1022  detectorNum : `int`, optional
1023  The integer number for the detector to process.
1024  isGen3 : bool, optional
1025  Flag this call to run() as using the Gen3 butler environment.
1026  strayLightData : `object`, optional
1027  Opaque object containing calibration information for stray-light
1028  correction. If `None`, no correction will be performed.
1029 
1030  Returns
1031  -------
1032  result : `lsst.pipe.base.Struct`
1033  Result struct with component:
1034  - ``exposure`` : `afw.image.Exposure`
1035  The fully ISR corrected exposure.
1036  - ``outputExposure`` : `afw.image.Exposure`
1037  An alias for `exposure`
1038  - ``ossThumb`` : `numpy.ndarray`
1039  Thumbnail image of the exposure after overscan subtraction.
1040  - ``flattenedThumb`` : `numpy.ndarray`
1041  Thumbnail image of the exposure after flat-field correction.
1042 
1043  Raises
1044  ------
1045  RuntimeError
1046  Raised if a configuration option is set to True, but the
1047  required calibration data has not been specified.
1048 
1049  Notes
1050  -----
1051  The current processed exposure can be viewed by setting the
1052  appropriate lsstDebug entries in the `debug.display`
1053  dictionary. The names of these entries correspond to some of
1054  the IsrTaskConfig Boolean options, with the value denoting the
1055  frame to use. The exposure is shown inside the matching
1056  option check and after the processing of that step has
1057  finished. The steps with debug points are:
1058 
1059  doAssembleCcd
1060  doBias
1061  doCrosstalk
1062  doBrighterFatter
1063  doDark
1064  doFringe
1065  doStrayLight
1066  doFlat
1067 
1068  In addition, setting the "postISRCCD" entry displays the
1069  exposure after all ISR processing has finished.
1070 
1071  """
1072 
1073  if isGen3 is True:
1074  # Gen3 currently cannot automatically do configuration overrides.
1075  # DM-15257 looks to discuss this issue.
1076 
1077  self.config.doFringe = False
1078 
1079  # Configure input exposures;
1080  if detectorNum is None:
1081  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1082 
1083  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1084  bias = self.ensureExposure(bias, camera, detectorNum)
1085  dark = self.ensureExposure(dark, camera, detectorNum)
1086  flat = self.ensureExposure(flat, camera, detectorNum)
1087  else:
1088  if isinstance(ccdExposure, ButlerDataRef):
1089  return self.runDataRef(ccdExposure)
1090 
1091  ccd = ccdExposure.getDetector()
1092 
1093  if not ccd:
1094  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1095  ccd = [FakeAmp(ccdExposure, self.config)]
1096 
1097  # Validate Input
1098  if self.config.doBias and bias is None:
1099  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1100  if self.doLinearize(ccd) and linearizer is None:
1101  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1102  if self.config.doBrighterFatter and bfKernel is None:
1103  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1104  if self.config.doDark and dark is None:
1105  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1106  if fringes is None:
1107  fringes = pipeBase.Struct(fringes=None)
1108  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1109  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1110  if self.config.doFlat and flat is None:
1111  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1112  if self.config.doDefect and defects is None:
1113  raise RuntimeError("Must supply defects if config.doDefect=True.")
1114  if self.config.doAddDistortionModel and camera is None:
1115  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1116 
1117  # Begin ISR processing.
1118  if self.config.doConvertIntToFloat:
1119  self.log.info("Converting exposure to floating point values")
1120  ccdExposure = self.convertIntToFloat(ccdExposure)
1121 
1122  # Amplifier level processing.
1123  overscans = []
1124  for amp in ccd:
1125  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1126  if ccdExposure.getBBox().contains(amp.getBBox()):
1127  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1128  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1129 
1130  if self.config.doOverscan and not badAmp:
1131  # Overscan correction on amp-by-amp basis.
1132  overscanResults = self.overscanCorrection(ccdExposure, amp)
1133  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1134  if self.config.qa is not None and self.config.qa.saveStats is True:
1135  if isinstance(overscanResults.overscanFit, float):
1136  qaMedian = overscanResults.overscanFit
1137  qaStdev = float("NaN")
1138  else:
1139  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1140  afwMath.MEDIAN | afwMath.STDEVCLIP)
1141  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1142  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1143 
1144  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1145  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1146  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1147  (amp.getName(), qaMedian, qaStdev))
1148  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1149  else:
1150  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1151  overscanResults = None
1152 
1153  overscans.append(overscanResults if overscanResults is not None else None)
1154  else:
1155  self.log.info("Skipped OSCAN")
1156 
1157  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1158  self.log.info("Applying crosstalk correction.")
1159  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1160  self.debugView(ccdExposure, "doCrosstalk")
1161 
1162  if self.config.doAssembleCcd:
1163  self.log.info("Assembling CCD from amplifiers")
1164  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1165 
1166  if self.config.expectWcs and not ccdExposure.getWcs():
1167  self.log.warn("No WCS found in input exposure")
1168  self.debugView(ccdExposure, "doAssembleCcd")
1169 
1170  ossThumb = None
1171  if self.config.qa.doThumbnailOss:
1172  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1173 
1174  if self.config.doBias:
1175  self.log.info("Applying bias correction.")
1176  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1177  trimToFit=self.config.doTrimToMatchCalib)
1178  self.debugView(ccdExposure, "doBias")
1179 
1180  if self.config.doVariance:
1181  for amp, overscanResults in zip(ccd, overscans):
1182  if ccdExposure.getBBox().contains(amp.getBBox()):
1183  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1184  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1185  if overscanResults is not None:
1186  self.updateVariance(ampExposure, amp,
1187  overscanImage=overscanResults.overscanImage)
1188  else:
1189  self.updateVariance(ampExposure, amp,
1190  overscanImage=None)
1191  if self.config.qa is not None and self.config.qa.saveStats is True:
1192  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1193  afwMath.MEDIAN | afwMath.STDEVCLIP)
1194  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1195  qaStats.getValue(afwMath.MEDIAN))
1196  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1197  qaStats.getValue(afwMath.STDEVCLIP))
1198  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1199  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1200  qaStats.getValue(afwMath.STDEVCLIP)))
1201 
1202  if self.doLinearize(ccd):
1203  self.log.info("Applying linearizer.")
1204  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1205 
1206  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1207  self.log.info("Applying crosstalk correction.")
1208  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1209  self.debugView(ccdExposure, "doCrosstalk")
1210 
1211  if self.config.doWidenSaturationTrails:
1212  self.log.info("Widening saturation trails.")
1213  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1214 
1215  interpolationDone = False
1216  if self.config.doBrighterFatter:
1217  # We need to apply flats and darks before we can interpolate, and we
1218  # need to interpolate before we do B-F, but we do B-F without the
1219  # flats and darks applied so we can work in units of electrons or holes.
1220  # This context manager applies and then removes the darks and flats.
1221  with self.flatContext(ccdExposure, flat, dark):
1222  if self.config.doDefect:
1223  self.maskAndInterpDefect(ccdExposure, defects)
1224 
1225  if self.config.doSaturationInterpolation:
1226  self.saturationInterpolation(ccdExposure)
1227 
1228  self.maskAndInterpNan(ccdExposure)
1229  interpolationDone = True
1230 
1231  self.log.info("Applying brighter fatter correction.")
1232  isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1233  self.config.brighterFatterMaxIter,
1234  self.config.brighterFatterThreshold,
1235  self.config.brighterFatterApplyGain,
1236  )
1237  self.debugView(ccdExposure, "doBrighterFatter")
1238 
1239  if self.config.doDark:
1240  self.log.info("Applying dark correction.")
1241  self.darkCorrection(ccdExposure, dark)
1242  self.debugView(ccdExposure, "doDark")
1243 
1244  if self.config.doFringe and not self.config.fringeAfterFlat:
1245  self.log.info("Applying fringe correction before flat.")
1246  self.fringe.run(ccdExposure, **fringes.getDict())
1247  self.debugView(ccdExposure, "doFringe")
1248 
1249  if self.config.doStrayLight:
1250  if strayLightData is not None:
1251  self.log.info("Applying stray light correction.")
1252  self.strayLight.run(ccdExposure, strayLightData)
1253  self.debugView(ccdExposure, "doStrayLight")
1254  else:
1255  self.log.debug("Skipping stray light correction: no data found for this image.")
1256 
1257  if self.config.doFlat:
1258  self.log.info("Applying flat correction.")
1259  self.flatCorrection(ccdExposure, flat)
1260  self.debugView(ccdExposure, "doFlat")
1261 
1262  if self.config.doApplyGains:
1263  self.log.info("Applying gain correction instead of flat.")
1264  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1265 
1266  if self.config.doDefect and not interpolationDone:
1267  self.log.info("Masking and interpolating defects.")
1268  self.maskAndInterpDefect(ccdExposure, defects)
1269 
1270  if self.config.doSaturationInterpolation and not interpolationDone:
1271  self.log.info("Interpolating saturated pixels.")
1272  self.saturationInterpolation(ccdExposure)
1273 
1274  if self.config.doNanInterpAfterFlat or not interpolationDone:
1275  self.log.info("Masking and interpolating NAN value pixels.")
1276  self.maskAndInterpNan(ccdExposure)
1277 
1278  if self.config.doFringe and self.config.fringeAfterFlat:
1279  self.log.info("Applying fringe correction after flat.")
1280  self.fringe.run(ccdExposure, **fringes.getDict())
1281 
1282  if self.config.doSetBadRegions:
1283  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1284  if badPixelCount > 0:
1285  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1286 
1287  flattenedThumb = None
1288  if self.config.qa.doThumbnailFlattened:
1289  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1290 
1291  if self.config.doCameraSpecificMasking:
1292  self.log.info("Masking regions for camera specific reasons.")
1293  self.masking.run(ccdExposure)
1294 
1295  self.roughZeroPoint(ccdExposure)
1296 
1297  if self.config.doVignette:
1298  self.log.info("Constructing Vignette polygon.")
1299  self.vignettePolygon = self.vignette.run(ccdExposure)
1300 
1301  if self.config.vignette.doWriteVignettePolygon:
1302  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1303 
1304  if self.config.doAttachTransmissionCurve:
1305  self.log.info("Adding transmission curves.")
1306  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1307  filterTransmission=filterTransmission,
1308  sensorTransmission=sensorTransmission,
1309  atmosphereTransmission=atmosphereTransmission)
1310 
1311  if self.config.doAddDistortionModel:
1312  self.log.info("Adding a distortion model to the WCS.")
1313  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1314 
1315  if self.config.doMeasureBackground:
1316  self.log.info("Measuring background level:")
1317  self.measureBackground(ccdExposure, self.config.qa)
1318 
1319  if self.config.qa is not None and self.config.qa.saveStats is True:
1320  for amp in ccd:
1321  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1322  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1323  afwMath.MEDIAN | afwMath.STDEVCLIP)
1324  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1325  qaStats.getValue(afwMath.MEDIAN))
1326  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1327  qaStats.getValue(afwMath.STDEVCLIP))
1328  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1329  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1330  qaStats.getValue(afwMath.STDEVCLIP)))
1331 
1332  self.debugView(ccdExposure, "postISRCCD")
1333 
1334  return pipeBase.Struct(
1335  exposure=ccdExposure,
1336  ossThumb=ossThumb,
1337  flattenedThumb=flattenedThumb,
1338 
1339  outputExposure=ccdExposure,
1340  outputOssThumbnail=ossThumb,
1341  outputFlattenedThumbnail=flattenedThumb,
1342  )
1343 
1344  @pipeBase.timeMethod
1345  def runDataRef(self, sensorRef):
1346  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1347 
1348  This method contains the `CmdLineTask` interface to the ISR
1349  processing. All IO is handled here, freeing the `run()` method
1350  to manage only pixel-level calculations. The steps performed
1351  are:
1352  - Read in necessary detrending/isr/calibration data.
1353  - Process raw exposure in `run()`.
1354  - Persist the ISR-corrected exposure as "postISRCCD" if
1355  config.doWrite=True.
1356 
1357  Parameters
1358  ----------
1359  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1360  DataRef of the detector data to be processed
1361 
1362  Returns
1363  -------
1364  result : `lsst.pipe.base.Struct`
1365  Result struct with component:
1366  - ``exposure`` : `afw.image.Exposure`
1367  The fully ISR corrected exposure.
1368 
1369  Raises
1370  ------
1371  RuntimeError
1372  Raised if a configuration option is set to True, but the
1373  required calibration data does not exist.
1374 
1375  """
1376  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1377 
1378  ccdExposure = sensorRef.get(self.config.datasetType)
1379 
1380  camera = sensorRef.get("camera")
1381  if camera is None and self.config.doAddDistortionModel:
1382  raise RuntimeError("config.doAddDistortionModel is True "
1383  "but could not get a camera from the butler")
1384  isrData = self.readIsrData(sensorRef, ccdExposure)
1385 
1386  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1387 
1388  if self.config.doWrite:
1389  sensorRef.put(result.exposure, "postISRCCD")
1390  if result.ossThumb is not None:
1391  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1392  if result.flattenedThumb is not None:
1393  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1394 
1395  return result
1396 
1397  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1398  """!Retrieve a calibration dataset for removing instrument signature.
1399 
1400  Parameters
1401  ----------
1402 
1403  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1404  DataRef of the detector data to find calibration datasets
1405  for.
1406  datasetType : `str`
1407  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1408  immediate : `Bool`
1409  If True, disable butler proxies to enable error handling
1410  within this routine.
1411 
1412  Returns
1413  -------
1414  exposure : `lsst.afw.image.Exposure`
1415  Requested calibration frame.
1416 
1417  Raises
1418  ------
1419  RuntimeError
1420  Raised if no matching calibration frame can be found.
1421  """
1422  try:
1423  exp = dataRef.get(datasetType, immediate=immediate)
1424  except Exception as exc1:
1425  if not self.config.fallbackFilterName:
1426  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1427  try:
1428  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1429  except Exception as exc2:
1430  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1431  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1432  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1433 
1434  if self.config.doAssembleIsrExposures:
1435  exp = self.assembleCcd.assembleCcd(exp)
1436  return exp
1437 
1438  def ensureExposure(self, inputExp, camera, detectorNum):
1439  """Ensure that the data returned by Butler is a fully constructed exposure.
1440 
1441  ISR requires exposure-level image data for historical reasons, so if we did
1442  not recieve that from Butler, construct it from what we have, modifying the
1443  input in place.
1444 
1445  Parameters
1446  ----------
1447  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1448  `lsst.afw.image.ImageF`
1449  The input data structure obtained from Butler.
1450  camera : `lsst.afw.cameraGeom.camera`
1451  The camera associated with the image. Used to find the appropriate
1452  detector.
1453  detectorNum : `int`
1454  The detector this exposure should match.
1455 
1456  Returns
1457  -------
1458  inputExp : `lsst.afw.image.Exposure`
1459  The re-constructed exposure, with appropriate detector parameters.
1460 
1461  Raises
1462  ------
1463  TypeError
1464  Raised if the input data cannot be used to construct an exposure.
1465  """
1466  if isinstance(inputExp, afwImage.DecoratedImageU):
1467  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1468  elif isinstance(inputExp, afwImage.ImageF):
1469  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1470  elif isinstance(inputExp, afwImage.MaskedImageF):
1471  inputExp = afwImage.makeExposure(inputExp)
1472  elif isinstance(inputExp, afwImage.Exposure):
1473  pass
1474  else:
1475  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1476 
1477  if inputExp.getDetector() is None:
1478  inputExp.setDetector(camera[detectorNum])
1479 
1480  return inputExp
1481 
1482  def convertIntToFloat(self, exposure):
1483  """Convert exposure image from uint16 to float.
1484 
1485  If the exposure does not need to be converted, the input is
1486  immediately returned. For exposures that are converted to use
1487  floating point pixels, the variance is set to unity and the
1488  mask to zero.
1489 
1490  Parameters
1491  ----------
1492  exposure : `lsst.afw.image.Exposure`
1493  The raw exposure to be converted.
1494 
1495  Returns
1496  -------
1497  newexposure : `lsst.afw.image.Exposure`
1498  The input ``exposure``, converted to floating point pixels.
1499 
1500  Raises
1501  ------
1502  RuntimeError
1503  Raised if the exposure type cannot be converted to float.
1504 
1505  """
1506  if isinstance(exposure, afwImage.ExposureF):
1507  # Nothing to be done
1508  return exposure
1509  if not hasattr(exposure, "convertF"):
1510  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1511 
1512  newexposure = exposure.convertF()
1513  newexposure.variance[:] = 1
1514  newexposure.mask[:] = 0x0
1515 
1516  return newexposure
1517 
1518  def maskAmplifier(self, ccdExposure, amp, defects):
1519  """Identify bad amplifiers, saturated and suspect pixels.
1520 
1521  Parameters
1522  ----------
1523  ccdExposure : `lsst.afw.image.Exposure`
1524  Input exposure to be masked.
1525  amp : `lsst.afw.table.AmpInfoCatalog`
1526  Catalog of parameters defining the amplifier on this
1527  exposure to mask.
1528  defects : `list`
1529  List of defects. Used to determine if the entire
1530  amplifier is bad.
1531 
1532  Returns
1533  -------
1534  badAmp : `Bool`
1535  If this is true, the entire amplifier area is covered by
1536  defects and unusable.
1537 
1538  """
1539  maskedImage = ccdExposure.getMaskedImage()
1540 
1541  badAmp = False
1542 
1543  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1544  # comparison with current defects definition.
1545  if defects is not None:
1546  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1547 
1548  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1549  # association with pixels in current ccdExposure).
1550  if badAmp:
1551  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1552  afwImage.PARENT)
1553  maskView = dataView.getMask()
1554  maskView |= maskView.getPlaneBitMask("BAD")
1555  del maskView
1556  return badAmp
1557 
1558  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1559  # Saturation and suspect pixels can be masked now, though.
1560  limits = dict()
1561  if self.config.doSaturation and not badAmp:
1562  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1563  if self.config.doSuspect and not badAmp:
1564  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1565 
1566  for maskName, maskThreshold in limits.items():
1567  if not math.isnan(maskThreshold):
1568  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1569  isrFunctions.makeThresholdMask(
1570  maskedImage=dataView,
1571  threshold=maskThreshold,
1572  growFootprints=0,
1573  maskName=maskName
1574  )
1575 
1576  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1577  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1578  afwImage.PARENT)
1579  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1580  self.config.suspectMaskName])
1581  if numpy.all(maskView.getArray() & maskVal > 0):
1582  badAmp = True
1583 
1584  return badAmp
1585 
1586  def overscanCorrection(self, ccdExposure, amp):
1587  """Apply overscan correction in place.
1588 
1589  This method does initial pixel rejection of the overscan
1590  region. The overscan can also be optionally segmented to
1591  allow for discontinuous overscan responses to be fit
1592  separately. The actual overscan subtraction is performed by
1593  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1594  which is called here after the amplifier is preprocessed.
1595 
1596  Parameters
1597  ----------
1598  ccdExposure : `lsst.afw.image.Exposure`
1599  Exposure to have overscan correction performed.
1600  amp : `lsst.afw.table.AmpInfoCatalog`
1601  The amplifier to consider while correcting the overscan.
1602 
1603  Returns
1604  -------
1605  overscanResults : `lsst.pipe.base.Struct`
1606  Result struct with components:
1607  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1608  Value or fit subtracted from the amplifier image data.
1609  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1610  Value or fit subtracted from the overscan image data.
1611  - ``overscanImage`` : `lsst.afw.image.Image`
1612  Image of the overscan region with the overscan
1613  correction applied. This quantity is used to estimate
1614  the amplifier read noise empirically.
1615 
1616  Raises
1617  ------
1618  RuntimeError
1619  Raised if the ``amp`` does not contain raw pixel information.
1620 
1621  See Also
1622  --------
1623  lsst.ip.isr.isrFunctions.overscanCorrection
1624  """
1625  if not amp.getHasRawInfo():
1626  raise RuntimeError("This method must be executed on an amp with raw information.")
1627 
1628  if amp.getRawHorizontalOverscanBBox().isEmpty():
1629  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1630  return None
1631 
1632  statControl = afwMath.StatisticsControl()
1633  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1634 
1635  # Determine the bounding boxes
1636  dataBBox = amp.getRawDataBBox()
1637  oscanBBox = amp.getRawHorizontalOverscanBBox()
1638  dx0 = 0
1639  dx1 = 0
1640 
1641  prescanBBox = amp.getRawPrescanBBox()
1642  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1643  dx0 += self.config.overscanNumLeadingColumnsToSkip
1644  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1645  else:
1646  dx0 += self.config.overscanNumTrailingColumnsToSkip
1647  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1648 
1649  # Determine if we need to work on subregions of the amplifier and overscan.
1650  imageBBoxes = []
1651  overscanBBoxes = []
1652 
1653  if ((self.config.overscanBiasJump and
1654  self.config.overscanBiasJumpLocation) and
1655  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1656  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1657  self.config.overscanBiasJumpDevices)):
1658  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1659  yLower = self.config.overscanBiasJumpLocation
1660  yUpper = dataBBox.getHeight() - yLower
1661  else:
1662  yUpper = self.config.overscanBiasJumpLocation
1663  yLower = dataBBox.getHeight() - yUpper
1664 
1665  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1666  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1667  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1668  afwGeom.Extent2I(dx0, 0),
1669  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1670  yLower)))
1671 
1672  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1673  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1674  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1675  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1676  yUpper)))
1677  else:
1678  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1679  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1680  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1681  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1682  oscanBBox.getHeight())))
1683 
1684  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1685  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1686  ampImage = ccdExposure.maskedImage[imageBBox]
1687  overscanImage = ccdExposure.maskedImage[overscanBBox]
1688 
1689  overscanArray = overscanImage.image.array
1690  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1691  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1692  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1693 
1694  statControl = afwMath.StatisticsControl()
1695  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1696 
1697  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1698  overscanImage=overscanImage,
1699  fitType=self.config.overscanFitType,
1700  order=self.config.overscanOrder,
1701  collapseRej=self.config.overscanNumSigmaClip,
1702  statControl=statControl,
1703  overscanIsInt=self.config.overscanIsInt
1704  )
1705 
1706  # Measure average overscan levels and record them in the metadata
1707  levelStat = afwMath.MEDIAN
1708  sigmaStat = afwMath.STDEVCLIP
1709 
1710  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1711  self.config.qa.flatness.nIter)
1712  metadata = ccdExposure.getMetadata()
1713  ampNum = amp.getName()
1714  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1715  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1716  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1717  else:
1718  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1719  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1720  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1721 
1722  return overscanResults
1723 
1724  def updateVariance(self, ampExposure, amp, overscanImage=None):
1725  """Set the variance plane using the amplifier gain and read noise
1726 
1727  The read noise is calculated from the ``overscanImage`` if the
1728  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1729  the value from the amplifier data is used.
1730 
1731  Parameters
1732  ----------
1733  ampExposure : `lsst.afw.image.Exposure`
1734  Exposure to process.
1735  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1736  Amplifier detector data.
1737  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1738  Image of overscan, required only for empirical read noise.
1739 
1740  See also
1741  --------
1742  lsst.ip.isr.isrFunctions.updateVariance
1743  """
1744  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1745  gain = amp.getGain()
1746 
1747  if math.isnan(gain):
1748  gain = 1.0
1749  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1750  elif gain <= 0:
1751  patchedGain = 1.0
1752  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1753  (amp.getName(), gain, patchedGain))
1754  gain = patchedGain
1755 
1756  if self.config.doEmpiricalReadNoise and overscanImage is None:
1757  self.log.info("Overscan is none for EmpiricalReadNoise")
1758 
1759  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1760  stats = afwMath.StatisticsControl()
1761  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1762  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1763  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1764  else:
1765  readNoise = amp.getReadNoise()
1766 
1767  isrFunctions.updateVariance(
1768  maskedImage=ampExposure.getMaskedImage(),
1769  gain=gain,
1770  readNoise=readNoise,
1771  )
1772 
1773  def darkCorrection(self, exposure, darkExposure, invert=False):
1774  """!Apply dark correction in place.
1775 
1776  Parameters
1777  ----------
1778  exposure : `lsst.afw.image.Exposure`
1779  Exposure to process.
1780  darkExposure : `lsst.afw.image.Exposure`
1781  Dark exposure of the same size as ``exposure``.
1782  invert : `Bool`, optional
1783  If True, re-add the dark to an already corrected image.
1784 
1785  Raises
1786  ------
1787  RuntimeError
1788  Raised if either ``exposure`` or ``darkExposure`` do not
1789  have their dark time defined.
1790 
1791  See Also
1792  --------
1793  lsst.ip.isr.isrFunctions.darkCorrection
1794  """
1795  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1796  if math.isnan(expScale):
1797  raise RuntimeError("Exposure darktime is NAN")
1798  if darkExposure.getInfo().getVisitInfo() is not None:
1799  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1800  else:
1801  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1802  # so getDarkTime() does not exist.
1803  darkScale = 1.0
1804 
1805  if math.isnan(darkScale):
1806  raise RuntimeError("Dark calib darktime is NAN")
1807  isrFunctions.darkCorrection(
1808  maskedImage=exposure.getMaskedImage(),
1809  darkMaskedImage=darkExposure.getMaskedImage(),
1810  expScale=expScale,
1811  darkScale=darkScale,
1812  invert=invert,
1813  trimToFit=self.config.doTrimToMatchCalib
1814  )
1815 
1816  def doLinearize(self, detector):
1817  """!Check if linearization is needed for the detector cameraGeom.
1818 
1819  Checks config.doLinearize and the linearity type of the first
1820  amplifier.
1821 
1822  Parameters
1823  ----------
1824  detector : `lsst.afw.cameraGeom.Detector`
1825  Detector to get linearity type from.
1826 
1827  Returns
1828  -------
1829  doLinearize : `Bool`
1830  If True, linearization should be performed.
1831  """
1832  return self.config.doLinearize and \
1833  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1834 
1835  def flatCorrection(self, exposure, flatExposure, invert=False):
1836  """!Apply flat correction in place.
1837 
1838  Parameters
1839  ----------
1840  exposure : `lsst.afw.image.Exposure`
1841  Exposure to process.
1842  flatExposure : `lsst.afw.image.Exposure`
1843  Flat exposure of the same size as ``exposure``.
1844  invert : `Bool`, optional
1845  If True, unflatten an already flattened image.
1846 
1847  See Also
1848  --------
1849  lsst.ip.isr.isrFunctions.flatCorrection
1850  """
1851  isrFunctions.flatCorrection(
1852  maskedImage=exposure.getMaskedImage(),
1853  flatMaskedImage=flatExposure.getMaskedImage(),
1854  scalingType=self.config.flatScalingType,
1855  userScale=self.config.flatUserScale,
1856  invert=invert,
1857  trimToFit=self.config.doTrimToMatchCalib
1858  )
1859 
1860  def saturationDetection(self, exposure, amp):
1861  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1862 
1863  Parameters
1864  ----------
1865  exposure : `lsst.afw.image.Exposure`
1866  Exposure to process. Only the amplifier DataSec is processed.
1867  amp : `lsst.afw.table.AmpInfoCatalog`
1868  Amplifier detector data.
1869 
1870  See Also
1871  --------
1872  lsst.ip.isr.isrFunctions.makeThresholdMask
1873  """
1874  if not math.isnan(amp.getSaturation()):
1875  maskedImage = exposure.getMaskedImage()
1876  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1877  isrFunctions.makeThresholdMask(
1878  maskedImage=dataView,
1879  threshold=amp.getSaturation(),
1880  growFootprints=0,
1881  maskName=self.config.saturatedMaskName,
1882  )
1883 
1884  def saturationInterpolation(self, ccdExposure):
1885  """!Interpolate over saturated pixels, in place.
1886 
1887  This method should be called after `saturationDetection`, to
1888  ensure that the saturated pixels have been identified in the
1889  SAT mask. It should also be called after `assembleCcd`, since
1890  saturated regions may cross amplifier boundaries.
1891 
1892  Parameters
1893  ----------
1894  exposure : `lsst.afw.image.Exposure`
1895  Exposure to process.
1896 
1897  See Also
1898  --------
1899  lsst.ip.isr.isrTask.saturationDetection
1900  lsst.ip.isr.isrFunctions.interpolateFromMask
1901  """
1902  isrFunctions.interpolateFromMask(
1903  maskedImage=ccdExposure.getMaskedImage(),
1904  fwhm=self.config.fwhm,
1905  growFootprints=self.config.growSaturationFootprintSize,
1906  maskName=self.config.saturatedMaskName,
1907  )
1908 
1909  def suspectDetection(self, exposure, amp):
1910  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1911 
1912  Parameters
1913  ----------
1914  exposure : `lsst.afw.image.Exposure`
1915  Exposure to process. Only the amplifier DataSec is processed.
1916  amp : `lsst.afw.table.AmpInfoCatalog`
1917  Amplifier detector data.
1918 
1919  See Also
1920  --------
1921  lsst.ip.isr.isrFunctions.makeThresholdMask
1922 
1923  Notes
1924  -----
1925  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1926  This is intended to indicate pixels that may be affected by unknown systematics;
1927  for example if non-linearity corrections above a certain level are unstable
1928  then that would be a useful value for suspectLevel. A value of `nan` indicates
1929  that no such level exists and no pixels are to be masked as suspicious.
1930  """
1931  suspectLevel = amp.getSuspectLevel()
1932  if math.isnan(suspectLevel):
1933  return
1934 
1935  maskedImage = exposure.getMaskedImage()
1936  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1937  isrFunctions.makeThresholdMask(
1938  maskedImage=dataView,
1939  threshold=suspectLevel,
1940  growFootprints=0,
1941  maskName=self.config.suspectMaskName,
1942  )
1943 
1944  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1945  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1946 
1947  Parameters
1948  ----------
1949  ccdExposure : `lsst.afw.image.Exposure`
1950  Exposure to process.
1951  defectBaseList : `List`
1952  List of defects to mask and interpolate.
1953 
1954  Notes
1955  -----
1956  Call this after CCD assembly, since defects may cross amplifier boundaries.
1957  """
1958  maskedImage = ccdExposure.getMaskedImage()
1959  defectList = []
1960  for d in defectBaseList:
1961  bbox = d.getBBox()
1962  nd = measAlg.Defect(bbox)
1963  defectList.append(nd)
1964  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1965  isrFunctions.interpolateDefectList(
1966  maskedImage=maskedImage,
1967  defectList=defectList,
1968  fwhm=self.config.fwhm,
1969  )
1970 
1971  if self.config.numEdgeSuspect > 0:
1972  goodBBox = maskedImage.getBBox()
1973  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1974  goodBBox.grow(-self.config.numEdgeSuspect)
1975  # Mask pixels outside goodBBox as SUSPECT
1976  SourceDetectionTask.setEdgeBits(
1977  maskedImage,
1978  goodBBox,
1979  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1980  )
1981 
1982  def maskAndInterpNan(self, exposure):
1983  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1984 
1985  Parameters
1986  ----------
1987  exposure : `lsst.afw.image.Exposure`
1988  Exposure to process.
1989 
1990  Notes
1991  -----
1992  We mask and interpolate over all NaNs, including those
1993  that are masked with other bits (because those may or may
1994  not be interpolated over later, and we want to remove all
1995  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1996  is used to preserve the historical name.
1997  """
1998  maskedImage = exposure.getMaskedImage()
1999 
2000  # Find and mask NaNs
2001  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2002  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2003  numNans = maskNans(maskedImage, maskVal)
2004  self.metadata.set("NUMNANS", numNans)
2005 
2006  # Interpolate over these previously-unmasked NaNs
2007  if numNans > 0:
2008  self.log.warn("There were %i unmasked NaNs", numNans)
2009  nanDefectList = isrFunctions.getDefectListFromMask(
2010  maskedImage=maskedImage,
2011  maskName='UNMASKEDNAN',
2012  )
2013  isrFunctions.interpolateDefectList(
2014  maskedImage=exposure.getMaskedImage(),
2015  defectList=nanDefectList,
2016  fwhm=self.config.fwhm,
2017  )
2018 
2019  def measureBackground(self, exposure, IsrQaConfig=None):
2020  """Measure the image background in subgrids, for quality control purposes.
2021 
2022  Parameters
2023  ----------
2024  exposure : `lsst.afw.image.Exposure`
2025  Exposure to process.
2026  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2027  Configuration object containing parameters on which background
2028  statistics and subgrids to use.
2029  """
2030  if IsrQaConfig is not None:
2031  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2032  IsrQaConfig.flatness.nIter)
2033  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2034  statsControl.setAndMask(maskVal)
2035  maskedImage = exposure.getMaskedImage()
2036  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2037  skyLevel = stats.getValue(afwMath.MEDIAN)
2038  skySigma = stats.getValue(afwMath.STDEVCLIP)
2039  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2040  metadata = exposure.getMetadata()
2041  metadata.set('SKYLEVEL', skyLevel)
2042  metadata.set('SKYSIGMA', skySigma)
2043 
2044  # calcluating flatlevel over the subgrids
2045  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2046  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2047  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2048  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2049  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2050  skyLevels = numpy.zeros((nX, nY))
2051 
2052  for j in range(nY):
2053  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2054  for i in range(nX):
2055  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2056 
2057  xLLC = xc - meshXHalf
2058  yLLC = yc - meshYHalf
2059  xURC = xc + meshXHalf - 1
2060  yURC = yc + meshYHalf - 1
2061 
2062  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2063  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2064 
2065  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2066 
2067  good = numpy.where(numpy.isfinite(skyLevels))
2068  skyMedian = numpy.median(skyLevels[good])
2069  flatness = (skyLevels[good] - skyMedian) / skyMedian
2070  flatness_rms = numpy.std(flatness)
2071  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2072 
2073  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2074  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2075  (nX, nY, flatness_pp, flatness_rms))
2076 
2077  metadata.set('FLATNESS_PP', float(flatness_pp))
2078  metadata.set('FLATNESS_RMS', float(flatness_rms))
2079  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2080  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2081  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2082 
2083  def roughZeroPoint(self, exposure):
2084  """Set an approximate magnitude zero point for the exposure.
2085 
2086  Parameters
2087  ----------
2088  exposure : `lsst.afw.image.Exposure`
2089  Exposure to process.
2090  """
2091  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2092  if filterName in self.config.fluxMag0T1:
2093  fluxMag0 = self.config.fluxMag0T1[filterName]
2094  else:
2095  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2096  fluxMag0 = self.config.defaultFluxMag0T1
2097 
2098  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2099  if not expTime > 0: # handle NaN as well as <= 0
2100  self.log.warn("Non-positive exposure time; skipping rough zero point")
2101  return
2102 
2103  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2104  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2105 
2106  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2107  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2108 
2109  Parameters
2110  ----------
2111  ccdExposure : `lsst.afw.image.Exposure`
2112  Exposure to process.
2113  fpPolygon : `lsst.afw.geom.Polygon`
2114  Polygon in focal plane coordinates.
2115  """
2116  # Get ccd corners in focal plane coordinates
2117  ccd = ccdExposure.getDetector()
2118  fpCorners = ccd.getCorners(FOCAL_PLANE)
2119  ccdPolygon = Polygon(fpCorners)
2120 
2121  # Get intersection of ccd corners with fpPolygon
2122  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2123 
2124  # Transform back to pixel positions and build new polygon
2125  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2126  validPolygon = Polygon(ccdPoints)
2127  ccdExposure.getInfo().setValidPolygon(validPolygon)
2128 
2129  @contextmanager
2130  def flatContext(self, exp, flat, dark=None):
2131  """Context manager that applies and removes flats and darks,
2132  if the task is configured to apply them.
2133 
2134  Parameters
2135  ----------
2136  exp : `lsst.afw.image.Exposure`
2137  Exposure to process.
2138  flat : `lsst.afw.image.Exposure`
2139  Flat exposure the same size as ``exp``.
2140  dark : `lsst.afw.image.Exposure`, optional
2141  Dark exposure the same size as ``exp``.
2142 
2143  Yields
2144  ------
2145  exp : `lsst.afw.image.Exposure`
2146  The flat and dark corrected exposure.
2147  """
2148  if self.config.doDark and dark is not None:
2149  self.darkCorrection(exp, dark)
2150  if self.config.doFlat:
2151  self.flatCorrection(exp, flat)
2152  try:
2153  yield exp
2154  finally:
2155  if self.config.doFlat:
2156  self.flatCorrection(exp, flat, invert=True)
2157  if self.config.doDark and dark is not None:
2158  self.darkCorrection(exp, dark, invert=True)
2159 
2160  def debugView(self, exposure, stepname):
2161  """Utility function to examine ISR exposure at different stages.
2162 
2163  Parameters
2164  ----------
2165  exposure : `lsst.afw.image.Exposure`
2166  Exposure to view.
2167  stepname : `str`
2168  State of processing to view.
2169  """
2170  frame = getDebugFrame(self._display, stepname)
2171  if frame:
2172  display = getDisplay(frame)
2173  display.scale('asinh', 'zscale')
2174  display.mtv(exposure)
2175 
2176 
2177 class FakeAmp(object):
2178  """A Detector-like object that supports returning gain and saturation level
2179 
2180  This is used when the input exposure does not have a detector.
2181 
2182  Parameters
2183  ----------
2184  exposure : `lsst.afw.image.Exposure`
2185  Exposure to generate a fake amplifier for.
2186  config : `lsst.ip.isr.isrTaskConfig`
2187  Configuration to apply to the fake amplifier.
2188  """
2189 
2190  def __init__(self, exposure, config):
2191  self._bbox = exposure.getBBox(afwImage.LOCAL)
2192  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
2193  self._gain = config.gain
2194  self._readNoise = config.readNoise
2195  self._saturation = config.saturation
2196 
2197  def getBBox(self):
2198  return self._bbox
2199 
2200  def getRawBBox(self):
2201  return self._bbox
2202 
2203  def getHasRawInfo(self):
2204  return True # but see getRawHorizontalOverscanBBox()
2205 
2207  return self._RawHorizontalOverscanBBox
2208 
2209  def getGain(self):
2210  return self._gain
2211 
2212  def getReadNoise(self):
2213  return self._readNoise
2214 
2215  def getSaturation(self):
2216  return self._saturation
2217 
2218  def getSuspectLevel(self):
2219  return float("NaN")
2220 
2221 
2222 class RunIsrConfig(pexConfig.Config):
2223  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2224 
2225 
2226 class RunIsrTask(pipeBase.CmdLineTask):
2227  """Task to wrap the default IsrTask to allow it to be retargeted.
2228 
2229  The standard IsrTask can be called directly from a command line
2230  program, but doing so removes the ability of the task to be
2231  retargeted. As most cameras override some set of the IsrTask
2232  methods, this would remove those data-specific methods in the
2233  output post-ISR images. This wrapping class fixes the issue,
2234  allowing identical post-ISR images to be generated by both the
2235  processCcd and isrTask code.
2236  """
2237  ConfigClass = RunIsrConfig
2238  _DefaultName = "runIsr"
2239 
2240  def __init__(self, *args, **kwargs):
2241  super().__init__(*args, **kwargs)
2242  self.makeSubtask("isr")
2243 
2244  def runDataRef(self, dataRef):
2245  """
2246  Parameters
2247  ----------
2248  dataRef : `lsst.daf.persistence.ButlerDataRef`
2249  data reference of the detector data to be processed
2250 
2251  Returns
2252  -------
2253  result : `pipeBase.Struct`
2254  Result struct with component:
2255 
2256  - exposure : `lsst.afw.image.Exposure`
2257  Post-ISR processed exposure.
2258  """
2259  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:711
def runDataRef(self, sensorRef)
Definition: isrTask.py:1345
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2019
def debugView(self, exposure, stepname)
Definition: isrTask.py:2160
def __init__(self, kwargs)
Definition: isrTask.py:701
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1438
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:824
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:778
def runDataRef(self, dataRef)
Definition: isrTask.py:2244
def __init__(self, args, kwargs)
Definition: isrTask.py:2240
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1982
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:761
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1884
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2083
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2206
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:748
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1586
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1482
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1835
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:821
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1397
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1773
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1816
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2106
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1518
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:771
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:962
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2130
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1724
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1909
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1944
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1860
def __init__(self, exposure, config)
Definition: isrTask.py:2190