lsst.ip.isr  17.0-5-g16fbfff+6
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defect
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "RunIsrTask"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="ExposureU",
78  dimensions=["Instrument", "Exposure", "Detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["Instrument", "CalibrationLabel"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["Instrument", "CalibrationLabel", "Detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["Instrument", "CalibrationLabel", "Detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["Instrument", "PhysicalFilter", "CalibrationLabel", "Detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["Instrument", "CalibrationLabel"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="Catalog",
120  dimensions=["Instrument", "CalibrationLabel", "Detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["Instrument", "CalibrationLabel"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["Instrument", "PhysicalFilter", "CalibrationLabel"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["Instrument", "CalibrationLabel", "Detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["Instrument"],
149  )
150 
151  # output datasets
152  outputExposure = pipeBase.OutputDatasetField(
153  doc="Output ISR processed exposure.",
154  name="postISRCCD",
155  scalar=True,
156  storageClass="ExposureF",
157  dimensions=["Instrument", "Visit", "Detector"],
158  )
159  outputOssThumbnail = pipeBase.OutputDatasetField(
160  doc="Output Overscan-subtracted thumbnail image.",
161  name="OssThumb",
162  scalar=True,
163  storageClass="Thumbnail",
164  dimensions=["Instrument", "Visit", "Detector"],
165  )
166  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167  doc="Output flat-corrected thumbnail image.",
168  name="FlattenedThumb",
169  scalar=True,
170  storageClass="TextStorage",
171  dimensions=["Instrument", "Visit", "Detector"],
172  )
173 
174  quantum = pipeBase.QuantumConfig(
175  dimensions=["Visit", "Detector", "Instrument"],
176  )
177 
178 
179  datasetType = pexConfig.Field(
180  dtype=str,
181  doc="Dataset type for input data; users will typically leave this alone, "
182  "but camera-specific ISR tasks will override it",
183  default="raw",
184  )
185 
186  fallbackFilterName = pexConfig.Field(
187  dtype=str,
188  doc="Fallback default filter name for calibrations.",
189  optional=True
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
195  )
196  fwhm = pexConfig.Field(
197  dtype=float,
198  doc="FWHM of PSF in arcseconds.",
199  default=1.0,
200  )
201  qa = pexConfig.ConfigField(
202  dtype=isrQa.IsrQaConfig,
203  doc="QA related configuration options.",
204  )
205 
206  # Image conversion configuration
207  doConvertIntToFloat = pexConfig.Field(
208  dtype=bool,
209  doc="Convert integer raw images to floating point values?",
210  default=True,
211  )
212 
213  # Saturated pixel handling.
214  doSaturation = pexConfig.Field(
215  dtype=bool,
216  doc="Mask saturated pixels? NB: this is totally independent of the"
217  " interpolation option - this is ONLY setting the bits in the mask."
218  " To have them interpolated make sure doSaturationInterpolation=True",
219  default=True,
220  )
221  saturatedMaskName = pexConfig.Field(
222  dtype=str,
223  doc="Name of mask plane to use in saturation detection and interpolation",
224  default="SAT",
225  )
226  saturation = pexConfig.Field(
227  dtype=float,
228  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229  default=float("NaN"),
230  )
231  growSaturationFootprintSize = pexConfig.Field(
232  dtype=int,
233  doc="Number of pixels by which to grow the saturation footprints",
234  default=1,
235  )
236 
237  # Suspect pixel handling.
238  doSuspect = pexConfig.Field(
239  dtype=bool,
240  doc="Mask suspect pixels?",
241  default=True,
242  )
243  suspectMaskName = pexConfig.Field(
244  dtype=str,
245  doc="Name of mask plane to use for suspect pixels",
246  default="SUSPECT",
247  )
248  numEdgeSuspect = pexConfig.Field(
249  dtype=int,
250  doc="Number of edge pixels to be flagged as untrustworthy.",
251  default=0,
252  )
253 
254  # Initial masking options.
255  doSetBadRegions = pexConfig.Field(
256  dtype=bool,
257  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
258  default=True,
259  )
260  badStatistic = pexConfig.ChoiceField(
261  dtype=str,
262  doc="How to estimate the average value for BAD regions.",
263  default='MEANCLIP',
264  allowed={
265  "MEANCLIP": "Correct using the (clipped) mean of good data",
266  "MEDIAN": "Correct using the median of the good data",
267  },
268  )
269 
270  # Overscan subtraction configuration.
271  doOverscan = pexConfig.Field(
272  dtype=bool,
273  doc="Do overscan subtraction?",
274  default=True,
275  )
276  overscanFitType = pexConfig.ChoiceField(
277  dtype=str,
278  doc="The method for fitting the overscan bias level.",
279  default='MEDIAN',
280  allowed={
281  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
282  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
283  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
284  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
285  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
286  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
287  "MEAN": "Correct using the mean of the overscan region",
288  "MEANCLIP": "Correct using a clipped mean of the overscan region",
289  "MEDIAN": "Correct using the median of the overscan region",
290  },
291  )
292  overscanOrder = pexConfig.Field(
293  dtype=int,
294  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
295  "or number of spline knots if overscan fit type is a spline."),
296  default=1,
297  )
298  overscanNumSigmaClip = pexConfig.Field(
299  dtype=float,
300  doc="Rejection threshold (sigma) for collapsing overscan before fit",
301  default=3.0,
302  )
303  overscanIsInt = pexConfig.Field(
304  dtype=bool,
305  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
306  default=True,
307  )
308  overscanNumLeadingColumnsToSkip = pexConfig.Field(
309  dtype=int,
310  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
311  default=0,
312  )
313  overscanNumTrailingColumnsToSkip = pexConfig.Field(
314  dtype=int,
315  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
316  default=0,
317  )
318  overscanMaxDev = pexConfig.Field(
319  dtype=float,
320  doc="Maximum deviation from the median for overscan",
321  default=1000.0, check=lambda x: x > 0
322  )
323  overscanBiasJump = pexConfig.Field(
324  dtype=bool,
325  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
326  default=False,
327  )
328  overscanBiasJumpKeyword = pexConfig.Field(
329  dtype=str,
330  doc="Header keyword containing information about devices.",
331  default="NO_SUCH_KEY",
332  )
333  overscanBiasJumpDevices = pexConfig.ListField(
334  dtype=str,
335  doc="List of devices that need piecewise overscan correction.",
336  default=(),
337  )
338  overscanBiasJumpLocation = pexConfig.Field(
339  dtype=int,
340  doc="Location of bias jump along y-axis.",
341  default=0,
342  )
343 
344  # Amplifier to CCD assembly configuration
345  doAssembleCcd = pexConfig.Field(
346  dtype=bool,
347  default=True,
348  doc="Assemble amp-level exposures into a ccd-level exposure?"
349  )
350  assembleCcd = pexConfig.ConfigurableField(
351  target=AssembleCcdTask,
352  doc="CCD assembly task",
353  )
354 
355  # General calibration configuration.
356  doAssembleIsrExposures = pexConfig.Field(
357  dtype=bool,
358  default=False,
359  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
360  )
361  doTrimToMatchCalib = pexConfig.Field(
362  dtype=bool,
363  default=False,
364  doc="Trim raw data to match calibration bounding boxes?"
365  )
366 
367  # Bias subtraction.
368  doBias = pexConfig.Field(
369  dtype=bool,
370  doc="Apply bias frame correction?",
371  default=True,
372  )
373  biasDataProductName = pexConfig.Field(
374  dtype=str,
375  doc="Name of the bias data product",
376  default="bias",
377  )
378 
379  # Variance construction
380  doVariance = pexConfig.Field(
381  dtype=bool,
382  doc="Calculate variance?",
383  default=True
384  )
385  gain = pexConfig.Field(
386  dtype=float,
387  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388  default=float("NaN"),
389  )
390  readNoise = pexConfig.Field(
391  dtype=float,
392  doc="The read noise to use if no Detector is present in the Exposure",
393  default=0.0,
394  )
395  doEmpiricalReadNoise = pexConfig.Field(
396  dtype=bool,
397  default=False,
398  doc="Calculate empirical read noise instead of value from AmpInfo data?"
399  )
400 
401  # Linearization.
402  doLinearize = pexConfig.Field(
403  dtype=bool,
404  doc="Correct for nonlinearity of the detector's response?",
405  default=True,
406  )
407 
408  # Crosstalk.
409  doCrosstalk = pexConfig.Field(
410  dtype=bool,
411  doc="Apply intra-CCD crosstalk correction?",
412  default=False,
413  )
414  doCrosstalkBeforeAssemble = pexConfig.Field(
415  dtype=bool,
416  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
417  default=True,
418  )
419  crosstalk = pexConfig.ConfigurableField(
420  target=CrosstalkTask,
421  doc="Intra-CCD crosstalk correction",
422  )
423 
424  # Masking option prior to brighter-fatter.
425  doWidenSaturationTrails = pexConfig.Field(
426  dtype=bool,
427  doc="Widen bleed trails based on their width?",
428  default=True
429  )
430 
431  # Brighter-Fatter correction.
432  doBrighterFatter = pexConfig.Field(
433  dtype=bool,
434  default=False,
435  doc="Apply the brighter fatter correction"
436  )
437  brighterFatterLevel = pexConfig.ChoiceField(
438  dtype=str,
439  default="DETECTOR",
440  doc="The level at which to correct for brighter-fatter.",
441  allowed={
442  "AMP": "Every amplifier treated separately.",
443  "DETECTOR": "One kernel per detector",
444  }
445  )
446  brighterFatterKernelFile = pexConfig.Field(
447  dtype=str,
448  default='',
449  doc="Kernel file used for the brighter fatter correction"
450  )
451  brighterFatterMaxIter = pexConfig.Field(
452  dtype=int,
453  default=10,
454  doc="Maximum number of iterations for the brighter fatter correction"
455  )
456  brighterFatterThreshold = pexConfig.Field(
457  dtype=float,
458  default=1000,
459  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
460  " absolute value of the difference between the current corrected image and the one"
461  " from the previous iteration summed over all the pixels."
462  )
463  brighterFatterApplyGain = pexConfig.Field(
464  dtype=bool,
465  default=True,
466  doc="Should the gain be applied when applying the brighter fatter correction?"
467  )
468 
469  # Defect and bad pixel correction options.
470  doDefect = pexConfig.Field(
471  dtype=bool,
472  doc="Apply correction for CCD defects, e.g. hot pixels?",
473  default=True,
474  )
475  doSaturationInterpolation = pexConfig.Field(
476  dtype=bool,
477  doc="Perform interpolation over pixels masked as saturated?"
478  " NB: This is independent of doSaturation; if that is False this plane"
479  " will likely be blank, resulting in a no-op here.",
480  default=True,
481  )
482  numEdgeSuspect = pexConfig.Field(
483  dtype=int,
484  doc="Number of edge pixels to be flagged as untrustworthy.",
485  default=0,
486  )
487 
488  # Dark subtraction.
489  doDark = pexConfig.Field(
490  dtype=bool,
491  doc="Apply dark frame correction?",
492  default=True,
493  )
494  darkDataProductName = pexConfig.Field(
495  dtype=str,
496  doc="Name of the dark data product",
497  default="dark",
498  )
499 
500  # Camera-specific stray light removal.
501  doStrayLight = pexConfig.Field(
502  dtype=bool,
503  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
504  default=False,
505  )
506  strayLight = pexConfig.ConfigurableField(
507  target=StrayLightTask,
508  doc="y-band stray light correction"
509  )
510 
511  # Flat correction.
512  doFlat = pexConfig.Field(
513  dtype=bool,
514  doc="Apply flat field correction?",
515  default=True,
516  )
517  flatDataProductName = pexConfig.Field(
518  dtype=str,
519  doc="Name of the flat data product",
520  default="flat",
521  )
522  flatScalingType = pexConfig.ChoiceField(
523  dtype=str,
524  doc="The method for scaling the flat on the fly.",
525  default='USER',
526  allowed={
527  "USER": "Scale by flatUserScale",
528  "MEAN": "Scale by the inverse of the mean",
529  "MEDIAN": "Scale by the inverse of the median",
530  },
531  )
532  flatUserScale = pexConfig.Field(
533  dtype=float,
534  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
535  default=1.0,
536  )
537  doTweakFlat = pexConfig.Field(
538  dtype=bool,
539  doc="Tweak flats to match observed amplifier ratios?",
540  default=False
541  )
542 
543  # Amplifier normalization based on gains instead of using flats configuration.
544  doApplyGains = pexConfig.Field(
545  dtype=bool,
546  doc="Correct the amplifiers for their gains instead of applying flat correction",
547  default=False,
548  )
549  normalizeGains = pexConfig.Field(
550  dtype=bool,
551  doc="Normalize all the amplifiers in each CCD to have the same median value.",
552  default=False,
553  )
554 
555  # Fringe correction.
556  doFringe = pexConfig.Field(
557  dtype=bool,
558  doc="Apply fringe correction?",
559  default=True,
560  )
561  fringe = pexConfig.ConfigurableField(
562  target=FringeTask,
563  doc="Fringe subtraction task",
564  )
565  fringeAfterFlat = pexConfig.Field(
566  dtype=bool,
567  doc="Do fringe subtraction after flat-fielding?",
568  default=True,
569  )
570 
571  # NAN pixel interpolation option.
572  doNanInterpAfterFlat = pexConfig.Field(
573  dtype=bool,
574  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
575  "also have to interpolate them before flat-fielding."),
576  default=False,
577  )
578 
579  # Distortion model application.
580  doAddDistortionModel = pexConfig.Field(
581  dtype=bool,
582  doc="Apply a distortion model based on camera geometry to the WCS?",
583  default=True,
584  )
585 
586  # Initial CCD-level background statistics options.
587  doMeasureBackground = pexConfig.Field(
588  dtype=bool,
589  doc="Measure the background level on the reduced image?",
590  default=False,
591  )
592 
593  # Camera-specific masking configuration.
594  doCameraSpecificMasking = pexConfig.Field(
595  dtype=bool,
596  doc="Mask camera-specific bad regions?",
597  default=False,
598  )
599  masking = pexConfig.ConfigurableField(
600  target=MaskingTask,
601  doc="Masking task."
602  )
603 
604  # Default photometric calibration options.
605  fluxMag0T1 = pexConfig.DictField(
606  keytype=str,
607  itemtype=float,
608  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
610  ))
611  )
612  defaultFluxMag0T1 = pexConfig.Field(
613  dtype=float,
614  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
615  default=pow(10.0, 0.4*28.0)
616  )
617 
618  # Vignette correction configuration.
619  doVignette = pexConfig.Field(
620  dtype=bool,
621  doc="Apply vignetting parameters?",
622  default=False,
623  )
624  vignette = pexConfig.ConfigurableField(
625  target=VignetteTask,
626  doc="Vignetting task.",
627  )
628 
629  # Transmission curve configuration.
630  doAttachTransmissionCurve = pexConfig.Field(
631  dtype=bool,
632  default=False,
633  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
634  )
635  doUseOpticsTransmission = pexConfig.Field(
636  dtype=bool,
637  default=True,
638  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
639  )
640  doUseFilterTransmission = pexConfig.Field(
641  dtype=bool,
642  default=True,
643  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
644  )
645  doUseSensorTransmission = pexConfig.Field(
646  dtype=bool,
647  default=True,
648  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
649  )
650  doUseAtmosphereTransmission = pexConfig.Field(
651  dtype=bool,
652  default=True,
653  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
654  )
655 
656  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
657  doWrite = pexConfig.Field(
658  dtype=bool,
659  doc="Persist postISRCCD?",
660  default=True,
661  )
662 
663  def validate(self):
664  super().validate()
665  if self.doFlat and self.doApplyGains:
666  raise ValueError("You may not specify both doFlat and doApplyGains")
667 
668 
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670  r"""Apply common instrument signature correction algorithms to a raw frame.
671 
672  The process for correcting imaging data is very similar from
673  camera to camera. This task provides a vanilla implementation of
674  doing these corrections, including the ability to turn certain
675  corrections off if they are not needed. The inputs to the primary
676  method, `run()`, are a raw exposure to be corrected and the
677  calibration data products. The raw input is a single chip sized
678  mosaic of all amps including overscans and other non-science
679  pixels. The method `runDataRef()` identifies and defines the
680  calibration data products, and is intended for use by a
681  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
682  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
683  subclassed for different camera, although the most camera specific
684  methods have been split into subtasks that can be redirected
685  appropriately.
686 
687  The __init__ method sets up the subtasks for ISR processing, using
688  the defaults from `lsst.ip.isr`.
689 
690  Parameters
691  ----------
692  args : `list`
693  Positional arguments passed to the Task constructor. None used at this time.
694  kwargs : `dict`, optional
695  Keyword arguments passed on to the Task constructor. None used at this time.
696  """
697  ConfigClass = IsrTaskConfig
698  _DefaultName = "isr"
699 
700  def __init__(self, **kwargs):
701  super().__init__(**kwargs)
702  self.makeSubtask("assembleCcd")
703  self.makeSubtask("crosstalk")
704  self.makeSubtask("strayLight")
705  self.makeSubtask("fringe")
706  self.makeSubtask("masking")
707  self.makeSubtask("vignette")
708 
709  @classmethod
710  def getInputDatasetTypes(cls, config):
711  inputTypeDict = super().getInputDatasetTypes(config)
712 
713  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
714  # need because the configuration tells us we will not be bothering with the
715  # correction that uses that IDT.
716  if config.doBias is not True:
717  inputTypeDict.pop("bias", None)
718  if config.doLinearize is not True:
719  inputTypeDict.pop("linearizer", None)
720  if config.doCrosstalk is not True:
721  inputTypeDict.pop("crosstalkSources", None)
722  if config.doBrighterFatter is not True:
723  inputTypeDict.pop("bfKernel", None)
724  if config.doDefect is not True:
725  inputTypeDict.pop("defects", None)
726  if config.doDark is not True:
727  inputTypeDict.pop("dark", None)
728  if config.doFlat is not True:
729  inputTypeDict.pop("flat", None)
730  if config.doAttachTransmissionCurve is not True:
731  inputTypeDict.pop("opticsTransmission", None)
732  inputTypeDict.pop("filterTransmission", None)
733  inputTypeDict.pop("sensorTransmission", None)
734  inputTypeDict.pop("atmosphereTransmission", None)
735  if config.doUseOpticsTransmission is not True:
736  inputTypeDict.pop("opticsTransmission", None)
737  if config.doUseFilterTransmission is not True:
738  inputTypeDict.pop("filterTransmission", None)
739  if config.doUseSensorTransmission is not True:
740  inputTypeDict.pop("sensorTransmission", None)
741  if config.doUseAtmosphereTransmission is not True:
742  inputTypeDict.pop("atmosphereTransmission", None)
743 
744  return inputTypeDict
745 
746  @classmethod
747  def getOutputDatasetTypes(cls, config):
748  outputTypeDict = super().getOutputDatasetTypes(config)
749 
750  if config.qa.doThumbnailOss is not True:
751  outputTypeDict.pop("outputOssThumbnail", None)
752  if config.qa.doThumbnailFlattened is not True:
753  outputTypeDict.pop("outputFlattenedThumbnail", None)
754  if config.doWrite is not True:
755  outputTypeDict.pop("outputExposure", None)
756 
757  return outputTypeDict
758 
759  @classmethod
760  def getPrerequisiteDatasetTypes(cls, config):
761  # Input calibration datasets should not constrain the QuantumGraph
762  # (it'd be confusing if not having flats just silently resulted in no
763  # data being processed). Our nomenclature for that is that these are
764  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
765  names = set(cls.getInputDatasetTypes(config))
766  names.remove("ccdExposure")
767  return names
768 
769  @classmethod
770  def getPerDatasetTypeDimensions(cls, config):
771  # Input calibration datasets of different types (i.e. flat, bias) need
772  # not have the same validity range. That makes CalibrationLabel
773  # (which maps directly to a validity range) a "per-DatasetType
774  # dimension".
775  return frozenset(["CalibrationLabel"])
776 
777  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
778  try:
779  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
780  except Exception as e:
781  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
782 
783  inputData['isGen3'] = True
784 
785  if self.config.doLinearize is True:
786  if 'linearizer' not in inputData.keys():
787  detector = inputData['camera'][inputData['detectorNum']]
788  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
789  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
790 
791  if inputData['defects'] is not None:
792  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
793  # masking expects a list of defects defined by their bounding box
794  defectList = []
795 
796  for r in inputData['defects']:
797  bbox = afwGeom.BoxI(afwGeom.PointI(r.get("x0"), r.get("y0")),
798  afwGeom.ExtentI(r.get("width"), r.get("height")))
799  defectList.append(Defect(bbox))
800 
801  inputData['defects'] = defectList
802 
803  # Broken: DM-17169
804  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
805  # fixed for non-HSC cameras in the future.
806  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
807  # if self.config.doCrosstalk else None)
808 
809  # Broken: DM-17152
810  # Fringes are not tested to be handled correctly by Gen3 butler.
811  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
812  # assembler=self.assembleCcd
813  # if self.config.doAssembleIsrExposures else None)
814  # if self.config.doFringe and
815  # self.fringe.checkFilter(inputData['ccdExposure'])
816  # else pipeBase.Struct(fringes=None))
817 
818  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
819 
820  def makeDatasetType(self, dsConfig):
821  return super().makeDatasetType(dsConfig)
822 
823  def readIsrData(self, dataRef, rawExposure):
824  """!Retrieve necessary frames for instrument signature removal.
825 
826  Pre-fetching all required ISR data products limits the IO
827  required by the ISR. Any conflict between the calibration data
828  available and that needed for ISR is also detected prior to
829  doing processing, allowing it to fail quickly.
830 
831  Parameters
832  ----------
833  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
834  Butler reference of the detector data to be processed
835  rawExposure : `afw.image.Exposure`
836  The raw exposure that will later be corrected with the
837  retrieved calibration data; should not be modified in this
838  method.
839 
840  Returns
841  -------
842  result : `lsst.pipe.base.Struct`
843  Result struct with components (which may be `None`):
844  - ``bias``: bias calibration frame (`afw.image.Exposure`)
845  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
846  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
847  - ``dark``: dark calibration frame (`afw.image.Exposure`)
848  - ``flat``: flat calibration frame (`afw.image.Exposure`)
849  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
850  - ``defects``: list of defects (`list`)
851  - ``fringes``: `lsst.pipe.base.Struct` with components:
852  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
853  - ``seed``: random seed derived from the ccdExposureId for random
854  number generator (`uint32`)
855  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
856  A ``TransmissionCurve`` that represents the throughput of the optics,
857  to be evaluated in focal-plane coordinates.
858  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
859  A ``TransmissionCurve`` that represents the throughput of the filter
860  itself, to be evaluated in focal-plane coordinates.
861  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
862  A ``TransmissionCurve`` that represents the throughput of the sensor
863  itself, to be evaluated in post-assembly trimmed detector coordinates.
864  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
865  A ``TransmissionCurve`` that represents the throughput of the
866  atmosphere, assumed to be spatially constant.
867 
868  """
869  ccd = rawExposure.getDetector()
870  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
871  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
872  if self.config.doBias else None)
873  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
874  linearizer = (dataRef.get("linearizer", immediate=True)
875  if self.doLinearize(ccd) else None)
876  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
877  if self.config.doCrosstalk else None)
878  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
879  if self.config.doDark else None)
880  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
881  if self.config.doFlat else None)
882  brighterFatterKernel = (dataRef.get("bfKernel")
883  if self.config.doBrighterFatter else None)
884  defectList = (dataRef.get("defects")
885  if self.config.doDefect else None)
886  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
887  if self.config.doAssembleIsrExposures else None)
888  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
889  else pipeBase.Struct(fringes=None))
890 
891  if self.config.doAttachTransmissionCurve:
892  opticsTransmission = (dataRef.get("transmission_optics")
893  if self.config.doUseOpticsTransmission else None)
894  filterTransmission = (dataRef.get("transmission_filter")
895  if self.config.doUseFilterTransmission else None)
896  sensorTransmission = (dataRef.get("transmission_sensor")
897  if self.config.doUseSensorTransmission else None)
898  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
899  if self.config.doUseAtmosphereTransmission else None)
900  else:
901  opticsTransmission = None
902  filterTransmission = None
903  sensorTransmission = None
904  atmosphereTransmission = None
905 
906  # Struct should include only kwargs to run()
907  return pipeBase.Struct(bias=biasExposure,
908  linearizer=linearizer,
909  crosstalkSources=crosstalkSources,
910  dark=darkExposure,
911  flat=flatExposure,
912  bfKernel=brighterFatterKernel,
913  defects=defectList,
914  fringes=fringeStruct,
915  opticsTransmission=opticsTransmission,
916  filterTransmission=filterTransmission,
917  sensorTransmission=sensorTransmission,
918  atmosphereTransmission=atmosphereTransmission,
919  )
920 
921  @pipeBase.timeMethod
922  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
923  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
924  opticsTransmission=None, filterTransmission=None,
925  sensorTransmission=None, atmosphereTransmission=None,
926  detectorNum=None, isGen3=False
927  ):
928  """!Perform instrument signature removal on an exposure.
929 
930  Steps included in the ISR processing, in order performed, are:
931  - saturation and suspect pixel masking
932  - overscan subtraction
933  - CCD assembly of individual amplifiers
934  - bias subtraction
935  - variance image construction
936  - linearization of non-linear response
937  - crosstalk masking
938  - brighter-fatter correction
939  - dark subtraction
940  - fringe correction
941  - stray light subtraction
942  - flat correction
943  - masking of known defects and camera specific features
944  - vignette calculation
945  - appending transmission curve and distortion model
946 
947  Parameters
948  ----------
949  ccdExposure : `lsst.afw.image.Exposure`
950  The raw exposure that is to be run through ISR. The
951  exposure is modified by this method.
952  camera : `lsst.afw.cameraGeom.Camera`, optional
953  The camera geometry for this exposure. Used to select the
954  distortion model appropriate for this data.
955  bias : `lsst.afw.image.Exposure`, optional
956  Bias calibration frame.
957  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
958  Functor for linearization.
959  crosstalkSources : `list`, optional
960  List of possible crosstalk sources.
961  dark : `lsst.afw.image.Exposure`, optional
962  Dark calibration frame.
963  flat : `lsst.afw.image.Exposure`, optional
964  Flat calibration frame.
965  bfKernel : `numpy.ndarray`, optional
966  Brighter-fatter kernel.
967  defects : `list`, optional
968  List of defects.
969  fringes : `lsst.pipe.base.Struct`, optional
970  Struct containing the fringe correction data, with
971  elements:
972  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
973  - ``seed``: random seed derived from the ccdExposureId for random
974  number generator (`uint32`)
975  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
976  A ``TransmissionCurve`` that represents the throughput of the optics,
977  to be evaluated in focal-plane coordinates.
978  filterTransmission : `lsst.afw.image.TransmissionCurve`
979  A ``TransmissionCurve`` that represents the throughput of the filter
980  itself, to be evaluated in focal-plane coordinates.
981  sensorTransmission : `lsst.afw.image.TransmissionCurve`
982  A ``TransmissionCurve`` that represents the throughput of the sensor
983  itself, to be evaluated in post-assembly trimmed detector coordinates.
984  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
985  A ``TransmissionCurve`` that represents the throughput of the
986  atmosphere, assumed to be spatially constant.
987  detectorNum : `int`, optional
988  The integer number for the detector to process.
989  isGen3 : bool, optional
990  Flag this call to run() as using the Gen3 butler environment.
991 
992  Returns
993  -------
994  result : `lsst.pipe.base.Struct`
995  Result struct with component:
996  - ``exposure`` : `afw.image.Exposure`
997  The fully ISR corrected exposure.
998  - ``outputExposure`` : `afw.image.Exposure`
999  An alias for `exposure`
1000  - ``ossThumb`` : `numpy.ndarray`
1001  Thumbnail image of the exposure after overscan subtraction.
1002  - ``flattenedThumb`` : `numpy.ndarray`
1003  Thumbnail image of the exposure after flat-field correction.
1004 
1005  Raises
1006  ------
1007  RuntimeError
1008  Raised if a configuration option is set to True, but the
1009  required calibration data has not been specified.
1010 
1011  Notes
1012  -----
1013  The current processed exposure can be viewed by setting the
1014  appropriate lsstDebug entries in the `debug.display`
1015  dictionary. The names of these entries correspond to some of
1016  the IsrTaskConfig Boolean options, with the value denoting the
1017  frame to use. The exposure is shown inside the matching
1018  option check and after the processing of that step has
1019  finished. The steps with debug points are:
1020 
1021  doAssembleCcd
1022  doBias
1023  doCrosstalk
1024  doBrighterFatter
1025  doDark
1026  doFringe
1027  doStrayLight
1028  doFlat
1029 
1030  In addition, setting the "postISRCCD" entry displays the
1031  exposure after all ISR processing has finished.
1032 
1033  """
1034 
1035  if isGen3 is True:
1036  # Gen3 currently cannot automatically do configuration overrides.
1037  # DM-15257 looks to discuss this issue.
1038 
1039  self.config.doFringe = False
1040 
1041  # Configure input exposures;
1042  if detectorNum is None:
1043  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1044 
1045  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1046  bias = self.ensureExposure(bias, camera, detectorNum)
1047  dark = self.ensureExposure(dark, camera, detectorNum)
1048  flat = self.ensureExposure(flat, camera, detectorNum)
1049  else:
1050  if isinstance(ccdExposure, ButlerDataRef):
1051  return self.runDataRef(ccdExposure)
1052 
1053  ccd = ccdExposure.getDetector()
1054 
1055  if not ccd:
1056  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1057  ccd = [FakeAmp(ccdExposure, self.config)]
1058 
1059  # Validate Input
1060  if self.config.doBias and bias is None:
1061  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1062  if self.doLinearize(ccd) and linearizer is None:
1063  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1064  if self.config.doBrighterFatter and bfKernel is None:
1065  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1066  if self.config.doDark and dark is None:
1067  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1068  if fringes is None:
1069  fringes = pipeBase.Struct(fringes=None)
1070  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1071  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1072  if self.config.doFlat and flat is None:
1073  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1074  if self.config.doDefect and defects is None:
1075  raise RuntimeError("Must supply defects if config.doDefect=True.")
1076  if self.config.doAddDistortionModel and camera is None:
1077  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1078 
1079  # Begin ISR processing.
1080  if self.config.doConvertIntToFloat:
1081  self.log.info("Converting exposure to floating point values")
1082  ccdExposure = self.convertIntToFloat(ccdExposure)
1083 
1084  # Amplifier level processing.
1085  overscans = []
1086  for amp in ccd:
1087  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1088  if ccdExposure.getBBox().contains(amp.getBBox()):
1089  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1090  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1091 
1092  if self.config.doOverscan and not badAmp:
1093  # Overscan correction on amp-by-amp basis.
1094  overscanResults = self.overscanCorrection(ccdExposure, amp)
1095  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1096  if self.config.qa is not None and self.config.qa.saveStats is True:
1097  if isinstance(overscanResults.overscanFit, float):
1098  qaMedian = overscanResults.overscanFit
1099  qaStdev = float("NaN")
1100  else:
1101  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1102  afwMath.MEDIAN | afwMath.STDEVCLIP)
1103  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1104  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1105 
1106  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1107  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1108  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1109  (amp.getName(), qaMedian, qaStdev))
1110  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1111  else:
1112  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1113  overscanResults = None
1114 
1115  overscans.append(overscanResults if overscanResults is not None else None)
1116  else:
1117  self.log.info("Skipped OSCAN")
1118 
1119  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1120  self.log.info("Applying crosstalk correction.")
1121  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1122  self.debugView(ccdExposure, "doCrosstalk")
1123 
1124  if self.config.doAssembleCcd:
1125  self.log.info("Assembling CCD from amplifiers")
1126  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1127 
1128  if self.config.expectWcs and not ccdExposure.getWcs():
1129  self.log.warn("No WCS found in input exposure")
1130  self.debugView(ccdExposure, "doAssembleCcd")
1131 
1132  ossThumb = None
1133  if self.config.qa.doThumbnailOss:
1134  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1135 
1136  if self.config.doBias:
1137  self.log.info("Applying bias correction.")
1138  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1139  trimToFit=self.config.doTrimToMatchCalib)
1140  self.debugView(ccdExposure, "doBias")
1141 
1142  if self.config.doVariance:
1143  for amp, overscanResults in zip(ccd, overscans):
1144  if ccdExposure.getBBox().contains(amp.getBBox()):
1145  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1146  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1147  if overscanResults is not None:
1148  self.updateVariance(ampExposure, amp,
1149  overscanImage=overscanResults.overscanImage)
1150  else:
1151  self.updateVariance(ampExposure, amp,
1152  overscanImage=None)
1153  if self.config.qa is not None and self.config.qa.saveStats is True:
1154  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1155  afwMath.MEDIAN | afwMath.STDEVCLIP)
1156  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1157  qaStats.getValue(afwMath.MEDIAN))
1158  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1159  qaStats.getValue(afwMath.STDEVCLIP))
1160  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1161  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1162  qaStats.getValue(afwMath.STDEVCLIP)))
1163 
1164  if self.doLinearize(ccd):
1165  self.log.info("Applying linearizer.")
1166  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1167 
1168  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1169  self.log.info("Applying crosstalk correction.")
1170  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1171  self.debugView(ccdExposure, "doCrosstalk")
1172 
1173  if self.config.doWidenSaturationTrails:
1174  self.log.info("Widening saturation trails.")
1175  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1176 
1177  interpolationDone = False
1178  if self.config.doBrighterFatter:
1179  # We need to apply flats and darks before we can interpolate, and we
1180  # need to interpolate before we do B-F, but we do B-F without the
1181  # flats and darks applied so we can work in units of electrons or holes.
1182  # This context manager applies and then removes the darks and flats.
1183  with self.flatContext(ccdExposure, flat, dark):
1184  if self.config.doDefect:
1185  self.maskAndInterpDefect(ccdExposure, defects)
1186 
1187  if self.config.doSaturationInterpolation:
1188  self.saturationInterpolation(ccdExposure)
1189 
1190  self.maskAndInterpNan(ccdExposure)
1191  interpolationDone = True
1192 
1193  if self.config.brighterFatterLevel == 'DETECTOR':
1194  kernelElement = bfKernel
1195  else:
1196  # TODO: DM-15631 for implementing this
1197  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
1198  self.log.info("Applying brighter fatter correction.")
1199  isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1200  self.config.brighterFatterMaxIter,
1201  self.config.brighterFatterThreshold,
1202  self.config.brighterFatterApplyGain,
1203  )
1204  self.debugView(ccdExposure, "doBrighterFatter")
1205 
1206  if self.config.doDark:
1207  self.log.info("Applying dark correction.")
1208  self.darkCorrection(ccdExposure, dark)
1209  self.debugView(ccdExposure, "doDark")
1210 
1211  if self.config.doFringe and not self.config.fringeAfterFlat:
1212  self.log.info("Applying fringe correction before flat.")
1213  self.fringe.run(ccdExposure, **fringes.getDict())
1214  self.debugView(ccdExposure, "doFringe")
1215 
1216  if self.config.doStrayLight:
1217  self.log.info("Applying stray light correction.")
1218  self.strayLight.run(ccdExposure)
1219  self.debugView(ccdExposure, "doStrayLight")
1220 
1221  if self.config.doFlat:
1222  self.log.info("Applying flat correction.")
1223  self.flatCorrection(ccdExposure, flat)
1224  self.debugView(ccdExposure, "doFlat")
1225 
1226  if self.config.doApplyGains:
1227  self.log.info("Applying gain correction instead of flat.")
1228  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1229 
1230  if self.config.doDefect and not interpolationDone:
1231  self.log.info("Masking and interpolating defects.")
1232  self.maskAndInterpDefect(ccdExposure, defects)
1233 
1234  if self.config.doSaturationInterpolation and not interpolationDone:
1235  self.log.info("Interpolating saturated pixels.")
1236  self.saturationInterpolation(ccdExposure)
1237 
1238  if self.config.doNanInterpAfterFlat or not interpolationDone:
1239  self.log.info("Masking and interpolating NAN value pixels.")
1240  self.maskAndInterpNan(ccdExposure)
1241 
1242  if self.config.doFringe and self.config.fringeAfterFlat:
1243  self.log.info("Applying fringe correction after flat.")
1244  self.fringe.run(ccdExposure, **fringes.getDict())
1245 
1246  if self.config.doSetBadRegions:
1247  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1248  if badPixelCount > 0:
1249  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1250 
1251  flattenedThumb = None
1252  if self.config.qa.doThumbnailFlattened:
1253  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1254 
1255  if self.config.doCameraSpecificMasking:
1256  self.log.info("Masking regions for camera specific reasons.")
1257  self.masking.run(ccdExposure)
1258 
1259  self.roughZeroPoint(ccdExposure)
1260 
1261  if self.config.doVignette:
1262  self.log.info("Constructing Vignette polygon.")
1263  self.vignettePolygon = self.vignette.run(ccdExposure)
1264 
1265  if self.config.vignette.doWriteVignettePolygon:
1266  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1267 
1268  if self.config.doAttachTransmissionCurve:
1269  self.log.info("Adding transmission curves.")
1270  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1271  filterTransmission=filterTransmission,
1272  sensorTransmission=sensorTransmission,
1273  atmosphereTransmission=atmosphereTransmission)
1274 
1275  if self.config.doAddDistortionModel:
1276  self.log.info("Adding a distortion model to the WCS.")
1277  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1278 
1279  if self.config.doMeasureBackground:
1280  self.log.info("Measuring background level:")
1281  self.measureBackground(ccdExposure, self.config.qa)
1282 
1283  if self.config.qa is not None and self.config.qa.saveStats is True:
1284  for amp in ccd:
1285  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1286  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1287  afwMath.MEDIAN | afwMath.STDEVCLIP)
1288  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1289  qaStats.getValue(afwMath.MEDIAN))
1290  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1291  qaStats.getValue(afwMath.STDEVCLIP))
1292  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1293  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1294  qaStats.getValue(afwMath.STDEVCLIP)))
1295 
1296  self.debugView(ccdExposure, "postISRCCD")
1297 
1298  return pipeBase.Struct(
1299  exposure=ccdExposure,
1300  ossThumb=ossThumb,
1301  flattenedThumb=flattenedThumb,
1302 
1303  outputExposure=ccdExposure,
1304  outputOssThumbnail=ossThumb,
1305  outputFlattenedThumbnail=flattenedThumb,
1306  )
1307 
1308  @pipeBase.timeMethod
1309  def runDataRef(self, sensorRef):
1310  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1311 
1312  This method contains the `CmdLineTask` interface to the ISR
1313  processing. All IO is handled here, freeing the `run()` method
1314  to manage only pixel-level calculations. The steps performed
1315  are:
1316  - Read in necessary detrending/isr/calibration data.
1317  - Process raw exposure in `run()`.
1318  - Persist the ISR-corrected exposure as "postISRCCD" if
1319  config.doWrite=True.
1320 
1321  Parameters
1322  ----------
1323  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1324  DataRef of the detector data to be processed
1325 
1326  Returns
1327  -------
1328  result : `lsst.pipe.base.Struct`
1329  Result struct with component:
1330  - ``exposure`` : `afw.image.Exposure`
1331  The fully ISR corrected exposure.
1332 
1333  Raises
1334  ------
1335  RuntimeError
1336  Raised if a configuration option is set to True, but the
1337  required calibration data does not exist.
1338 
1339  """
1340  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1341 
1342  ccdExposure = sensorRef.get(self.config.datasetType)
1343 
1344  camera = sensorRef.get("camera")
1345  if camera is None and self.config.doAddDistortionModel:
1346  raise RuntimeError("config.doAddDistortionModel is True "
1347  "but could not get a camera from the butler")
1348  isrData = self.readIsrData(sensorRef, ccdExposure)
1349 
1350  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1351 
1352  if self.config.doWrite:
1353  sensorRef.put(result.exposure, "postISRCCD")
1354  if result.ossThumb is not None:
1355  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1356  if result.flattenedThumb is not None:
1357  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1358 
1359  return result
1360 
1361  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1362  """!Retrieve a calibration dataset for removing instrument signature.
1363 
1364  Parameters
1365  ----------
1366 
1367  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1368  DataRef of the detector data to find calibration datasets
1369  for.
1370  datasetType : `str`
1371  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1372  immediate : `Bool`
1373  If True, disable butler proxies to enable error handling
1374  within this routine.
1375 
1376  Returns
1377  -------
1378  exposure : `lsst.afw.image.Exposure`
1379  Requested calibration frame.
1380 
1381  Raises
1382  ------
1383  RuntimeError
1384  Raised if no matching calibration frame can be found.
1385  """
1386  try:
1387  exp = dataRef.get(datasetType, immediate=immediate)
1388  except Exception as exc1:
1389  if not self.config.fallbackFilterName:
1390  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1391  try:
1392  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1393  except Exception as exc2:
1394  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1395  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1396  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1397 
1398  if self.config.doAssembleIsrExposures:
1399  exp = self.assembleCcd.assembleCcd(exp)
1400  return exp
1401 
1402  def ensureExposure(self, inputExp, camera, detectorNum):
1403  """Ensure that the data returned by Butler is a fully constructed exposure.
1404 
1405  ISR requires exposure-level image data for historical reasons, so if we did
1406  not recieve that from Butler, construct it from what we have, modifying the
1407  input in place.
1408 
1409  Parameters
1410  ----------
1411  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1412  `lsst.afw.image.ImageF`
1413  The input data structure obtained from Butler.
1414  camera : `lsst.afw.cameraGeom.camera`
1415  The camera associated with the image. Used to find the appropriate
1416  detector.
1417  detectorNum : `int`
1418  The detector this exposure should match.
1419 
1420  Returns
1421  -------
1422  inputExp : `lsst.afw.image.Exposure`
1423  The re-constructed exposure, with appropriate detector parameters.
1424 
1425  Raises
1426  ------
1427  TypeError
1428  Raised if the input data cannot be used to construct an exposure.
1429  """
1430  if isinstance(inputExp, afwImage.DecoratedImageU):
1431  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1432  elif isinstance(inputExp, afwImage.ImageF):
1433  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1434  elif isinstance(inputExp, afwImage.MaskedImageF):
1435  inputExp = afwImage.makeExposure(inputExp)
1436  elif isinstance(inputExp, afwImage.Exposure):
1437  pass
1438  else:
1439  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1440 
1441  if inputExp.getDetector() is None:
1442  inputExp.setDetector(camera[detectorNum])
1443 
1444  return inputExp
1445 
1446  def convertIntToFloat(self, exposure):
1447  """Convert exposure image from uint16 to float.
1448 
1449  If the exposure does not need to be converted, the input is
1450  immediately returned. For exposures that are converted to use
1451  floating point pixels, the variance is set to unity and the
1452  mask to zero.
1453 
1454  Parameters
1455  ----------
1456  exposure : `lsst.afw.image.Exposure`
1457  The raw exposure to be converted.
1458 
1459  Returns
1460  -------
1461  newexposure : `lsst.afw.image.Exposure`
1462  The input ``exposure``, converted to floating point pixels.
1463 
1464  Raises
1465  ------
1466  RuntimeError
1467  Raised if the exposure type cannot be converted to float.
1468 
1469  """
1470  if isinstance(exposure, afwImage.ExposureF):
1471  # Nothing to be done
1472  return exposure
1473  if not hasattr(exposure, "convertF"):
1474  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1475 
1476  newexposure = exposure.convertF()
1477  newexposure.variance[:] = 1
1478  newexposure.mask[:] = 0x0
1479 
1480  return newexposure
1481 
1482  def maskAmplifier(self, ccdExposure, amp, defects):
1483  """Identify bad amplifiers, saturated and suspect pixels.
1484 
1485  Parameters
1486  ----------
1487  ccdExposure : `lsst.afw.image.Exposure`
1488  Input exposure to be masked.
1489  amp : `lsst.afw.table.AmpInfoCatalog`
1490  Catalog of parameters defining the amplifier on this
1491  exposure to mask.
1492  defects : `list`
1493  List of defects. Used to determine if the entire
1494  amplifier is bad.
1495 
1496  Returns
1497  -------
1498  badAmp : `Bool`
1499  If this is true, the entire amplifier area is covered by
1500  defects and unusable.
1501 
1502  """
1503  maskedImage = ccdExposure.getMaskedImage()
1504 
1505  badAmp = False
1506 
1507  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1508  # comparison with current defects definition.
1509  if defects is not None:
1510  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1511 
1512  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1513  # association with pixels in current ccdExposure).
1514  if badAmp:
1515  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1516  afwImage.PARENT)
1517  maskView = dataView.getMask()
1518  maskView |= maskView.getPlaneBitMask("BAD")
1519  del maskView
1520  return badAmp
1521 
1522  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1523  # Saturation and suspect pixels can be masked now, though.
1524  limits = dict()
1525  if self.config.doSaturation and not badAmp:
1526  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1527  if self.config.doSuspect and not badAmp:
1528  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1529 
1530  for maskName, maskThreshold in limits.items():
1531  if not math.isnan(maskThreshold):
1532  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1533  isrFunctions.makeThresholdMask(
1534  maskedImage=dataView,
1535  threshold=maskThreshold,
1536  growFootprints=0,
1537  maskName=maskName
1538  )
1539 
1540  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1541  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1542  afwImage.PARENT)
1543  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1544  self.config.suspectMaskName])
1545  if numpy.all(maskView.getArray() & maskVal > 0):
1546  badAmp = True
1547 
1548  return badAmp
1549 
1550  def overscanCorrection(self, ccdExposure, amp):
1551  """Apply overscan correction in place.
1552 
1553  This method does initial pixel rejection of the overscan
1554  region. The overscan can also be optionally segmented to
1555  allow for discontinuous overscan responses to be fit
1556  separately. The actual overscan subtraction is performed by
1557  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1558  which is called here after the amplifier is preprocessed.
1559 
1560  Parameters
1561  ----------
1562  ccdExposure : `lsst.afw.image.Exposure`
1563  Exposure to have overscan correction performed.
1564  amp : `lsst.afw.table.AmpInfoCatalog`
1565  The amplifier to consider while correcting the overscan.
1566 
1567  Returns
1568  -------
1569  overscanResults : `lsst.pipe.base.Struct`
1570  Result struct with components:
1571  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1572  Value or fit subtracted from the amplifier image data.
1573  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1574  Value or fit subtracted from the overscan image data.
1575  - ``overscanImage`` : `lsst.afw.image.Image`
1576  Image of the overscan region with the overscan
1577  correction applied. This quantity is used to estimate
1578  the amplifier read noise empirically.
1579 
1580  Raises
1581  ------
1582  RuntimeError
1583  Raised if the ``amp`` does not contain raw pixel information.
1584 
1585  See Also
1586  --------
1587  lsst.ip.isr.isrFunctions.overscanCorrection
1588  """
1589  if not amp.getHasRawInfo():
1590  raise RuntimeError("This method must be executed on an amp with raw information.")
1591 
1592  if amp.getRawHorizontalOverscanBBox().isEmpty():
1593  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1594  return None
1595 
1596  statControl = afwMath.StatisticsControl()
1597  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1598 
1599  # Determine the bounding boxes
1600  dataBBox = amp.getRawDataBBox()
1601  oscanBBox = amp.getRawHorizontalOverscanBBox()
1602  dx0 = 0
1603  dx1 = 0
1604 
1605  prescanBBox = amp.getRawPrescanBBox()
1606  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1607  dx0 += self.config.overscanNumLeadingColumnsToSkip
1608  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1609  else:
1610  dx0 += self.config.overscanNumTrailingColumnsToSkip
1611  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1612 
1613  # Determine if we need to work on subregions of the amplifier and overscan.
1614  imageBBoxes = []
1615  overscanBBoxes = []
1616 
1617  if ((self.config.overscanBiasJump and
1618  self.config.overscanBiasJumpLocation) and
1619  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1620  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1621  self.config.overscanBiasJumpDevices)):
1622  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1623  yLower = self.config.overscanBiasJumpLocation
1624  yUpper = dataBBox.getHeight() - yLower
1625  else:
1626  yUpper = self.config.overscanBiasJumpLocation
1627  yLower = dataBBox.getHeight() - yUpper
1628 
1629  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1630  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1631  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1632  afwGeom.Extent2I(dx0, 0),
1633  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1634  yLower)))
1635 
1636  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1637  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1638  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1639  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1640  yUpper)))
1641  else:
1642  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1643  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1644  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1645  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1646  oscanBBox.getHeight())))
1647 
1648  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1649  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1650  ampImage = ccdExposure.maskedImage[imageBBox]
1651  overscanImage = ccdExposure.maskedImage[overscanBBox]
1652 
1653  overscanArray = overscanImage.image.array
1654  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1655  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1656  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1657 
1658  statControl = afwMath.StatisticsControl()
1659  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1660 
1661  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1662  overscanImage=overscanImage,
1663  fitType=self.config.overscanFitType,
1664  order=self.config.overscanOrder,
1665  collapseRej=self.config.overscanNumSigmaClip,
1666  statControl=statControl,
1667  overscanIsInt=self.config.overscanIsInt
1668  )
1669 
1670  # Measure average overscan levels and record them in the metadata
1671  levelStat = afwMath.MEDIAN
1672  sigmaStat = afwMath.STDEVCLIP
1673 
1674  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1675  self.config.qa.flatness.nIter)
1676  metadata = ccdExposure.getMetadata()
1677  ampNum = amp.getName()
1678  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1679  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1680  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1681  else:
1682  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1683  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1684  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1685 
1686  return overscanResults
1687 
1688  def updateVariance(self, ampExposure, amp, overscanImage=None):
1689  """Set the variance plane using the amplifier gain and read noise
1690 
1691  The read noise is calculated from the ``overscanImage`` if the
1692  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1693  the value from the amplifier data is used.
1694 
1695  Parameters
1696  ----------
1697  ampExposure : `lsst.afw.image.Exposure`
1698  Exposure to process.
1699  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1700  Amplifier detector data.
1701  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1702  Image of overscan, required only for empirical read noise.
1703 
1704  See also
1705  --------
1706  lsst.ip.isr.isrFunctions.updateVariance
1707  """
1708  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1709  gain = amp.getGain()
1710 
1711  if math.isnan(gain):
1712  gain = 1.0
1713  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1714  elif gain <= 0:
1715  patchedGain = 1.0
1716  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1717  (amp.getName(), gain, patchedGain))
1718  gain = patchedGain
1719 
1720  if self.config.doEmpiricalReadNoise and overscanImage is None:
1721  self.log.info("Overscan is none for EmpiricalReadNoise")
1722 
1723  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1724  stats = afwMath.StatisticsControl()
1725  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1726  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1727  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1728  else:
1729  readNoise = amp.getReadNoise()
1730 
1731  isrFunctions.updateVariance(
1732  maskedImage=ampExposure.getMaskedImage(),
1733  gain=gain,
1734  readNoise=readNoise,
1735  )
1736 
1737  def darkCorrection(self, exposure, darkExposure, invert=False):
1738  """!Apply dark correction in place.
1739 
1740  Parameters
1741  ----------
1742  exposure : `lsst.afw.image.Exposure`
1743  Exposure to process.
1744  darkExposure : `lsst.afw.image.Exposure`
1745  Dark exposure of the same size as ``exposure``.
1746  invert : `Bool`, optional
1747  If True, re-add the dark to an already corrected image.
1748 
1749  Raises
1750  ------
1751  RuntimeError
1752  Raised if either ``exposure`` or ``darkExposure`` do not
1753  have their dark time defined.
1754 
1755  See Also
1756  --------
1757  lsst.ip.isr.isrFunctions.darkCorrection
1758  """
1759  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1760  if math.isnan(expScale):
1761  raise RuntimeError("Exposure darktime is NAN")
1762  if darkExposure.getInfo().getVisitInfo() is not None:
1763  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1764  else:
1765  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1766  # so getDarkTime() does not exist.
1767  darkScale = 1.0
1768 
1769  if math.isnan(darkScale):
1770  raise RuntimeError("Dark calib darktime is NAN")
1771  isrFunctions.darkCorrection(
1772  maskedImage=exposure.getMaskedImage(),
1773  darkMaskedImage=darkExposure.getMaskedImage(),
1774  expScale=expScale,
1775  darkScale=darkScale,
1776  invert=invert,
1777  trimToFit=self.config.doTrimToMatchCalib
1778  )
1779 
1780  def doLinearize(self, detector):
1781  """!Check if linearization is needed for the detector cameraGeom.
1782 
1783  Checks config.doLinearize and the linearity type of the first
1784  amplifier.
1785 
1786  Parameters
1787  ----------
1788  detector : `lsst.afw.cameraGeom.Detector`
1789  Detector to get linearity type from.
1790 
1791  Returns
1792  -------
1793  doLinearize : `Bool`
1794  If True, linearization should be performed.
1795  """
1796  return self.config.doLinearize and \
1797  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1798 
1799  def flatCorrection(self, exposure, flatExposure, invert=False):
1800  """!Apply flat correction in place.
1801 
1802  Parameters
1803  ----------
1804  exposure : `lsst.afw.image.Exposure`
1805  Exposure to process.
1806  flatExposure : `lsst.afw.image.Exposure`
1807  Flat exposure of the same size as ``exposure``.
1808  invert : `Bool`, optional
1809  If True, unflatten an already flattened image.
1810 
1811  See Also
1812  --------
1813  lsst.ip.isr.isrFunctions.flatCorrection
1814  """
1815  isrFunctions.flatCorrection(
1816  maskedImage=exposure.getMaskedImage(),
1817  flatMaskedImage=flatExposure.getMaskedImage(),
1818  scalingType=self.config.flatScalingType,
1819  userScale=self.config.flatUserScale,
1820  invert=invert,
1821  trimToFit=self.config.doTrimToMatchCalib
1822  )
1823 
1824  def saturationDetection(self, exposure, amp):
1825  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1826 
1827  Parameters
1828  ----------
1829  exposure : `lsst.afw.image.Exposure`
1830  Exposure to process. Only the amplifier DataSec is processed.
1831  amp : `lsst.afw.table.AmpInfoCatalog`
1832  Amplifier detector data.
1833 
1834  See Also
1835  --------
1836  lsst.ip.isr.isrFunctions.makeThresholdMask
1837  """
1838  if not math.isnan(amp.getSaturation()):
1839  maskedImage = exposure.getMaskedImage()
1840  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1841  isrFunctions.makeThresholdMask(
1842  maskedImage=dataView,
1843  threshold=amp.getSaturation(),
1844  growFootprints=0,
1845  maskName=self.config.saturatedMaskName,
1846  )
1847 
1848  def saturationInterpolation(self, ccdExposure):
1849  """!Interpolate over saturated pixels, in place.
1850 
1851  This method should be called after `saturationDetection`, to
1852  ensure that the saturated pixels have been identified in the
1853  SAT mask. It should also be called after `assembleCcd`, since
1854  saturated regions may cross amplifier boundaries.
1855 
1856  Parameters
1857  ----------
1858  exposure : `lsst.afw.image.Exposure`
1859  Exposure to process.
1860 
1861  See Also
1862  --------
1863  lsst.ip.isr.isrTask.saturationDetection
1864  lsst.ip.isr.isrFunctions.interpolateFromMask
1865  """
1866  isrFunctions.interpolateFromMask(
1867  maskedImage=ccdExposure.getMaskedImage(),
1868  fwhm=self.config.fwhm,
1869  growFootprints=self.config.growSaturationFootprintSize,
1870  maskName=self.config.saturatedMaskName,
1871  )
1872 
1873  def suspectDetection(self, exposure, amp):
1874  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1875 
1876  Parameters
1877  ----------
1878  exposure : `lsst.afw.image.Exposure`
1879  Exposure to process. Only the amplifier DataSec is processed.
1880  amp : `lsst.afw.table.AmpInfoCatalog`
1881  Amplifier detector data.
1882 
1883  See Also
1884  --------
1885  lsst.ip.isr.isrFunctions.makeThresholdMask
1886 
1887  Notes
1888  -----
1889  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1890  This is intended to indicate pixels that may be affected by unknown systematics;
1891  for example if non-linearity corrections above a certain level are unstable
1892  then that would be a useful value for suspectLevel. A value of `nan` indicates
1893  that no such level exists and no pixels are to be masked as suspicious.
1894  """
1895  suspectLevel = amp.getSuspectLevel()
1896  if math.isnan(suspectLevel):
1897  return
1898 
1899  maskedImage = exposure.getMaskedImage()
1900  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1901  isrFunctions.makeThresholdMask(
1902  maskedImage=dataView,
1903  threshold=suspectLevel,
1904  growFootprints=0,
1905  maskName=self.config.suspectMaskName,
1906  )
1907 
1908  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1909  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1910 
1911  Parameters
1912  ----------
1913  ccdExposure : `lsst.afw.image.Exposure`
1914  Exposure to process.
1915  defectBaseList : `List`
1916  List of defects to mask and interpolate.
1917 
1918  Notes
1919  -----
1920  Call this after CCD assembly, since defects may cross amplifier boundaries.
1921  """
1922  maskedImage = ccdExposure.getMaskedImage()
1923  defectList = []
1924  for d in defectBaseList:
1925  bbox = d.getBBox()
1926  nd = measAlg.Defect(bbox)
1927  defectList.append(nd)
1928  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1929  isrFunctions.interpolateDefectList(
1930  maskedImage=maskedImage,
1931  defectList=defectList,
1932  fwhm=self.config.fwhm,
1933  )
1934 
1935  if self.config.numEdgeSuspect > 0:
1936  goodBBox = maskedImage.getBBox()
1937  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1938  goodBBox.grow(-self.config.numEdgeSuspect)
1939  # Mask pixels outside goodBBox as SUSPECT
1940  SourceDetectionTask.setEdgeBits(
1941  maskedImage,
1942  goodBBox,
1943  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1944  )
1945 
1946  def maskAndInterpNan(self, exposure):
1947  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1948 
1949  Parameters
1950  ----------
1951  exposure : `lsst.afw.image.Exposure`
1952  Exposure to process.
1953 
1954  Notes
1955  -----
1956  We mask and interpolate over all NaNs, including those
1957  that are masked with other bits (because those may or may
1958  not be interpolated over later, and we want to remove all
1959  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1960  is used to preserve the historical name.
1961  """
1962  maskedImage = exposure.getMaskedImage()
1963 
1964  # Find and mask NaNs
1965  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1966  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1967  numNans = maskNans(maskedImage, maskVal)
1968  self.metadata.set("NUMNANS", numNans)
1969 
1970  # Interpolate over these previously-unmasked NaNs
1971  if numNans > 0:
1972  self.log.warn("There were %i unmasked NaNs", numNans)
1973  nanDefectList = isrFunctions.getDefectListFromMask(
1974  maskedImage=maskedImage,
1975  maskName='UNMASKEDNAN',
1976  )
1977  isrFunctions.interpolateDefectList(
1978  maskedImage=exposure.getMaskedImage(),
1979  defectList=nanDefectList,
1980  fwhm=self.config.fwhm,
1981  )
1982 
1983  def measureBackground(self, exposure, IsrQaConfig=None):
1984  """Measure the image background in subgrids, for quality control purposes.
1985 
1986  Parameters
1987  ----------
1988  exposure : `lsst.afw.image.Exposure`
1989  Exposure to process.
1990  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
1991  Configuration object containing parameters on which background
1992  statistics and subgrids to use.
1993  """
1994  if IsrQaConfig is not None:
1995  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1996  IsrQaConfig.flatness.nIter)
1997  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
1998  statsControl.setAndMask(maskVal)
1999  maskedImage = exposure.getMaskedImage()
2000  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2001  skyLevel = stats.getValue(afwMath.MEDIAN)
2002  skySigma = stats.getValue(afwMath.STDEVCLIP)
2003  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2004  metadata = exposure.getMetadata()
2005  metadata.set('SKYLEVEL', skyLevel)
2006  metadata.set('SKYSIGMA', skySigma)
2007 
2008  # calcluating flatlevel over the subgrids
2009  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2010  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2011  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2012  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2013  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2014  skyLevels = numpy.zeros((nX, nY))
2015 
2016  for j in range(nY):
2017  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2018  for i in range(nX):
2019  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2020 
2021  xLLC = xc - meshXHalf
2022  yLLC = yc - meshYHalf
2023  xURC = xc + meshXHalf - 1
2024  yURC = yc + meshYHalf - 1
2025 
2026  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2027  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2028 
2029  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2030 
2031  good = numpy.where(numpy.isfinite(skyLevels))
2032  skyMedian = numpy.median(skyLevels[good])
2033  flatness = (skyLevels[good] - skyMedian) / skyMedian
2034  flatness_rms = numpy.std(flatness)
2035  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2036 
2037  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2038  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2039  (nX, nY, flatness_pp, flatness_rms))
2040 
2041  metadata.set('FLATNESS_PP', float(flatness_pp))
2042  metadata.set('FLATNESS_RMS', float(flatness_rms))
2043  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2044  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2045  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2046 
2047  def roughZeroPoint(self, exposure):
2048  """Set an approximate magnitude zero point for the exposure.
2049 
2050  Parameters
2051  ----------
2052  exposure : `lsst.afw.image.Exposure`
2053  Exposure to process.
2054  """
2055  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2056  if filterName in self.config.fluxMag0T1:
2057  fluxMag0 = self.config.fluxMag0T1[filterName]
2058  else:
2059  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2060  fluxMag0 = self.config.defaultFluxMag0T1
2061 
2062  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2063  if not expTime > 0: # handle NaN as well as <= 0
2064  self.log.warn("Non-positive exposure time; skipping rough zero point")
2065  return
2066 
2067  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2068  exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2069 
2070  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2071  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2072 
2073  Parameters
2074  ----------
2075  ccdExposure : `lsst.afw.image.Exposure`
2076  Exposure to process.
2077  fpPolygon : `lsst.afw.geom.Polygon`
2078  Polygon in focal plane coordinates.
2079  """
2080  # Get ccd corners in focal plane coordinates
2081  ccd = ccdExposure.getDetector()
2082  fpCorners = ccd.getCorners(FOCAL_PLANE)
2083  ccdPolygon = Polygon(fpCorners)
2084 
2085  # Get intersection of ccd corners with fpPolygon
2086  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2087 
2088  # Transform back to pixel positions and build new polygon
2089  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2090  validPolygon = Polygon(ccdPoints)
2091  ccdExposure.getInfo().setValidPolygon(validPolygon)
2092 
2093  @contextmanager
2094  def flatContext(self, exp, flat, dark=None):
2095  """Context manager that applies and removes flats and darks,
2096  if the task is configured to apply them.
2097 
2098  Parameters
2099  ----------
2100  exp : `lsst.afw.image.Exposure`
2101  Exposure to process.
2102  flat : `lsst.afw.image.Exposure`
2103  Flat exposure the same size as ``exp``.
2104  dark : `lsst.afw.image.Exposure`, optional
2105  Dark exposure the same size as ``exp``.
2106 
2107  Yields
2108  ------
2109  exp : `lsst.afw.image.Exposure`
2110  The flat and dark corrected exposure.
2111  """
2112  if self.config.doDark and dark is not None:
2113  self.darkCorrection(exp, dark)
2114  if self.config.doFlat:
2115  self.flatCorrection(exp, flat)
2116  try:
2117  yield exp
2118  finally:
2119  if self.config.doFlat:
2120  self.flatCorrection(exp, flat, invert=True)
2121  if self.config.doDark and dark is not None:
2122  self.darkCorrection(exp, dark, invert=True)
2123 
2124  def debugView(self, exposure, stepname):
2125  """Utility function to examine ISR exposure at different stages.
2126 
2127  Parameters
2128  ----------
2129  exposure : `lsst.afw.image.Exposure`
2130  Exposure to view.
2131  stepname : `str`
2132  State of processing to view.
2133  """
2134  frame = getDebugFrame(self._display, stepname)
2135  if frame:
2136  display = getDisplay(frame)
2137  display.scale('asinh', 'zscale')
2138  display.mtv(exposure)
2139 
2140 
2141 class FakeAmp(object):
2142  """A Detector-like object that supports returning gain and saturation level
2143 
2144  This is used when the input exposure does not have a detector.
2145 
2146  Parameters
2147  ----------
2148  exposure : `lsst.afw.image.Exposure`
2149  Exposure to generate a fake amplifier for.
2150  config : `lsst.ip.isr.isrTaskConfig`
2151  Configuration to apply to the fake amplifier.
2152  """
2153 
2154  def __init__(self, exposure, config):
2155  self._bbox = exposure.getBBox(afwImage.LOCAL)
2156  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
2157  self._gain = config.gain
2158  self._readNoise = config.readNoise
2159  self._saturation = config.saturation
2160 
2161  def getBBox(self):
2162  return self._bbox
2163 
2164  def getRawBBox(self):
2165  return self._bbox
2166 
2167  def getHasRawInfo(self):
2168  return True # but see getRawHorizontalOverscanBBox()
2169 
2171  return self._RawHorizontalOverscanBBox
2172 
2173  def getGain(self):
2174  return self._gain
2175 
2176  def getReadNoise(self):
2177  return self._readNoise
2178 
2179  def getSaturation(self):
2180  return self._saturation
2181 
2182  def getSuspectLevel(self):
2183  return float("NaN")
2184 
2185 
2186 class RunIsrConfig(pexConfig.Config):
2187  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2188 
2189 
2190 class RunIsrTask(pipeBase.CmdLineTask):
2191  """Task to wrap the default IsrTask to allow it to be retargeted.
2192 
2193  The standard IsrTask can be called directly from a command line
2194  program, but doing so removes the ability of the task to be
2195  retargeted. As most cameras override some set of the IsrTask
2196  methods, this would remove those data-specific methods in the
2197  output post-ISR images. This wrapping class fixes the issue,
2198  allowing identical post-ISR images to be generated by both the
2199  processCcd and isrTask code.
2200  """
2201  ConfigClass = RunIsrConfig
2202  _DefaultName = "runIsr"
2203 
2204  def __init__(self, *args, **kwargs):
2205  super().__init__(*args, **kwargs)
2206  self.makeSubtask("isr")
2207 
2208  def runDataRef(self, dataRef):
2209  """
2210  Parameters
2211  ----------
2212  dataRef : `lsst.daf.persistence.ButlerDataRef`
2213  data reference of the detector data to be processed
2214 
2215  Returns
2216  -------
2217  result : `pipeBase.Struct`
2218  Result struct with component:
2219 
2220  - exposure : `lsst.afw.image.Exposure`
2221  Post-ISR processed exposure.
2222  """
2223  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:710
def runDataRef(self, sensorRef)
Definition: isrTask.py:1309
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:1983
def debugView(self, exposure, stepname)
Definition: isrTask.py:2124
def __init__(self, kwargs)
Definition: isrTask.py:700
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1402
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:823
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:777
def runDataRef(self, dataRef)
Definition: isrTask.py:2208
def __init__(self, args, kwargs)
Definition: isrTask.py:2204
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1946
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:760
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1848
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2047
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2170
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:747
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1550
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1446
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1799
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:820
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1361
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1737
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1780
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:927
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2070
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1482
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:770
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2094
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1688
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1873
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1908
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1824
def __init__(self, exposure, config)
Definition: isrTask.py:2154