lsst.ip.isr  17.0.1-14-g2fe997d+2
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "RunIsrTask"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["instrument"],
149  )
150 
151  # output datasets
152  outputExposure = pipeBase.OutputDatasetField(
153  doc="Output ISR processed exposure.",
154  name="postISRCCD",
155  scalar=True,
156  storageClass="ExposureF",
157  dimensions=["instrument", "visit", "detector"],
158  )
159  outputOssThumbnail = pipeBase.OutputDatasetField(
160  doc="Output Overscan-subtracted thumbnail image.",
161  name="OssThumb",
162  scalar=True,
163  storageClass="Thumbnail",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167  doc="Output flat-corrected thumbnail image.",
168  name="FlattenedThumb",
169  scalar=True,
170  storageClass="TextStorage",
171  dimensions=["instrument", "visit", "detector"],
172  )
173 
174  quantum = pipeBase.QuantumConfig(
175  dimensions=["visit", "detector", "instrument"],
176  )
177 
178 
179  datasetType = pexConfig.Field(
180  dtype=str,
181  doc="Dataset type for input data; users will typically leave this alone, "
182  "but camera-specific ISR tasks will override it",
183  default="raw",
184  )
185 
186  fallbackFilterName = pexConfig.Field(
187  dtype=str,
188  doc="Fallback default filter name for calibrations.",
189  optional=True
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
195  )
196  fwhm = pexConfig.Field(
197  dtype=float,
198  doc="FWHM of PSF in arcseconds.",
199  default=1.0,
200  )
201  qa = pexConfig.ConfigField(
202  dtype=isrQa.IsrQaConfig,
203  doc="QA related configuration options.",
204  )
205 
206  # Image conversion configuration
207  doConvertIntToFloat = pexConfig.Field(
208  dtype=bool,
209  doc="Convert integer raw images to floating point values?",
210  default=True,
211  )
212 
213  # Saturated pixel handling.
214  doSaturation = pexConfig.Field(
215  dtype=bool,
216  doc="Mask saturated pixels? NB: this is totally independent of the"
217  " interpolation option - this is ONLY setting the bits in the mask."
218  " To have them interpolated make sure doSaturationInterpolation=True",
219  default=True,
220  )
221  saturatedMaskName = pexConfig.Field(
222  dtype=str,
223  doc="Name of mask plane to use in saturation detection and interpolation",
224  default="SAT",
225  )
226  saturation = pexConfig.Field(
227  dtype=float,
228  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229  default=float("NaN"),
230  )
231  growSaturationFootprintSize = pexConfig.Field(
232  dtype=int,
233  doc="Number of pixels by which to grow the saturation footprints",
234  default=1,
235  )
236 
237  # Suspect pixel handling.
238  doSuspect = pexConfig.Field(
239  dtype=bool,
240  doc="Mask suspect pixels?",
241  default=True,
242  )
243  suspectMaskName = pexConfig.Field(
244  dtype=str,
245  doc="Name of mask plane to use for suspect pixels",
246  default="SUSPECT",
247  )
248  numEdgeSuspect = pexConfig.Field(
249  dtype=int,
250  doc="Number of edge pixels to be flagged as untrustworthy.",
251  default=0,
252  )
253 
254  # Initial masking options.
255  doSetBadRegions = pexConfig.Field(
256  dtype=bool,
257  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
258  default=True,
259  )
260  badStatistic = pexConfig.ChoiceField(
261  dtype=str,
262  doc="How to estimate the average value for BAD regions.",
263  default='MEANCLIP',
264  allowed={
265  "MEANCLIP": "Correct using the (clipped) mean of good data",
266  "MEDIAN": "Correct using the median of the good data",
267  },
268  )
269 
270  # Overscan subtraction configuration.
271  doOverscan = pexConfig.Field(
272  dtype=bool,
273  doc="Do overscan subtraction?",
274  default=True,
275  )
276  overscanFitType = pexConfig.ChoiceField(
277  dtype=str,
278  doc="The method for fitting the overscan bias level.",
279  default='MEDIAN',
280  allowed={
281  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
282  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
283  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
284  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
285  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
286  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
287  "MEAN": "Correct using the mean of the overscan region",
288  "MEANCLIP": "Correct using a clipped mean of the overscan region",
289  "MEDIAN": "Correct using the median of the overscan region",
290  },
291  )
292  overscanOrder = pexConfig.Field(
293  dtype=int,
294  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
295  "or number of spline knots if overscan fit type is a spline."),
296  default=1,
297  )
298  overscanNumSigmaClip = pexConfig.Field(
299  dtype=float,
300  doc="Rejection threshold (sigma) for collapsing overscan before fit",
301  default=3.0,
302  )
303  overscanIsInt = pexConfig.Field(
304  dtype=bool,
305  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
306  default=True,
307  )
308  overscanNumLeadingColumnsToSkip = pexConfig.Field(
309  dtype=int,
310  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
311  default=0,
312  )
313  overscanNumTrailingColumnsToSkip = pexConfig.Field(
314  dtype=int,
315  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
316  default=0,
317  )
318  overscanMaxDev = pexConfig.Field(
319  dtype=float,
320  doc="Maximum deviation from the median for overscan",
321  default=1000.0, check=lambda x: x > 0
322  )
323  overscanBiasJump = pexConfig.Field(
324  dtype=bool,
325  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
326  default=False,
327  )
328  overscanBiasJumpKeyword = pexConfig.Field(
329  dtype=str,
330  doc="Header keyword containing information about devices.",
331  default="NO_SUCH_KEY",
332  )
333  overscanBiasJumpDevices = pexConfig.ListField(
334  dtype=str,
335  doc="List of devices that need piecewise overscan correction.",
336  default=(),
337  )
338  overscanBiasJumpLocation = pexConfig.Field(
339  dtype=int,
340  doc="Location of bias jump along y-axis.",
341  default=0,
342  )
343 
344  # Amplifier to CCD assembly configuration
345  doAssembleCcd = pexConfig.Field(
346  dtype=bool,
347  default=True,
348  doc="Assemble amp-level exposures into a ccd-level exposure?"
349  )
350  assembleCcd = pexConfig.ConfigurableField(
351  target=AssembleCcdTask,
352  doc="CCD assembly task",
353  )
354 
355  # General calibration configuration.
356  doAssembleIsrExposures = pexConfig.Field(
357  dtype=bool,
358  default=False,
359  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
360  )
361  doTrimToMatchCalib = pexConfig.Field(
362  dtype=bool,
363  default=False,
364  doc="Trim raw data to match calibration bounding boxes?"
365  )
366 
367  # Bias subtraction.
368  doBias = pexConfig.Field(
369  dtype=bool,
370  doc="Apply bias frame correction?",
371  default=True,
372  )
373  biasDataProductName = pexConfig.Field(
374  dtype=str,
375  doc="Name of the bias data product",
376  default="bias",
377  )
378 
379  # Variance construction
380  doVariance = pexConfig.Field(
381  dtype=bool,
382  doc="Calculate variance?",
383  default=True
384  )
385  gain = pexConfig.Field(
386  dtype=float,
387  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388  default=float("NaN"),
389  )
390  readNoise = pexConfig.Field(
391  dtype=float,
392  doc="The read noise to use if no Detector is present in the Exposure",
393  default=0.0,
394  )
395  doEmpiricalReadNoise = pexConfig.Field(
396  dtype=bool,
397  default=False,
398  doc="Calculate empirical read noise instead of value from AmpInfo data?"
399  )
400 
401  # Linearization.
402  doLinearize = pexConfig.Field(
403  dtype=bool,
404  doc="Correct for nonlinearity of the detector's response?",
405  default=True,
406  )
407 
408  # Crosstalk.
409  doCrosstalk = pexConfig.Field(
410  dtype=bool,
411  doc="Apply intra-CCD crosstalk correction?",
412  default=False,
413  )
414  doCrosstalkBeforeAssemble = pexConfig.Field(
415  dtype=bool,
416  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
417  default=True,
418  )
419  crosstalk = pexConfig.ConfigurableField(
420  target=CrosstalkTask,
421  doc="Intra-CCD crosstalk correction",
422  )
423 
424  # Masking option prior to brighter-fatter.
425  doWidenSaturationTrails = pexConfig.Field(
426  dtype=bool,
427  doc="Widen bleed trails based on their width?",
428  default=True
429  )
430 
431  # Brighter-Fatter correction.
432  doBrighterFatter = pexConfig.Field(
433  dtype=bool,
434  default=False,
435  doc="Apply the brighter fatter correction"
436  )
437  brighterFatterLevel = pexConfig.ChoiceField(
438  dtype=str,
439  default="DETECTOR",
440  doc="The level at which to correct for brighter-fatter.",
441  allowed={
442  "AMP": "Every amplifier treated separately.",
443  "DETECTOR": "One kernel per detector",
444  }
445  )
446  brighterFatterKernelFile = pexConfig.Field(
447  dtype=str,
448  default='',
449  doc="Kernel file used for the brighter fatter correction"
450  )
451  brighterFatterMaxIter = pexConfig.Field(
452  dtype=int,
453  default=10,
454  doc="Maximum number of iterations for the brighter fatter correction"
455  )
456  brighterFatterThreshold = pexConfig.Field(
457  dtype=float,
458  default=1000,
459  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
460  " absolute value of the difference between the current corrected image and the one"
461  " from the previous iteration summed over all the pixels."
462  )
463  brighterFatterApplyGain = pexConfig.Field(
464  dtype=bool,
465  default=True,
466  doc="Should the gain be applied when applying the brighter fatter correction?"
467  )
468 
469  # Defect and bad pixel correction options.
470  doDefect = pexConfig.Field(
471  dtype=bool,
472  doc="Apply correction for CCD defects, e.g. hot pixels?",
473  default=True,
474  )
475  doSaturationInterpolation = pexConfig.Field(
476  dtype=bool,
477  doc="Perform interpolation over pixels masked as saturated?"
478  " NB: This is independent of doSaturation; if that is False this plane"
479  " will likely be blank, resulting in a no-op here.",
480  default=True,
481  )
482  numEdgeSuspect = pexConfig.Field(
483  dtype=int,
484  doc="Number of edge pixels to be flagged as untrustworthy.",
485  default=0,
486  )
487 
488  # Dark subtraction.
489  doDark = pexConfig.Field(
490  dtype=bool,
491  doc="Apply dark frame correction?",
492  default=True,
493  )
494  darkDataProductName = pexConfig.Field(
495  dtype=str,
496  doc="Name of the dark data product",
497  default="dark",
498  )
499 
500  # Camera-specific stray light removal.
501  doStrayLight = pexConfig.Field(
502  dtype=bool,
503  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
504  default=False,
505  )
506  strayLight = pexConfig.ConfigurableField(
507  target=StrayLightTask,
508  doc="y-band stray light correction"
509  )
510 
511  # Flat correction.
512  doFlat = pexConfig.Field(
513  dtype=bool,
514  doc="Apply flat field correction?",
515  default=True,
516  )
517  flatDataProductName = pexConfig.Field(
518  dtype=str,
519  doc="Name of the flat data product",
520  default="flat",
521  )
522  flatScalingType = pexConfig.ChoiceField(
523  dtype=str,
524  doc="The method for scaling the flat on the fly.",
525  default='USER',
526  allowed={
527  "USER": "Scale by flatUserScale",
528  "MEAN": "Scale by the inverse of the mean",
529  "MEDIAN": "Scale by the inverse of the median",
530  },
531  )
532  flatUserScale = pexConfig.Field(
533  dtype=float,
534  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
535  default=1.0,
536  )
537  doTweakFlat = pexConfig.Field(
538  dtype=bool,
539  doc="Tweak flats to match observed amplifier ratios?",
540  default=False
541  )
542 
543  # Amplifier normalization based on gains instead of using flats configuration.
544  doApplyGains = pexConfig.Field(
545  dtype=bool,
546  doc="Correct the amplifiers for their gains instead of applying flat correction",
547  default=False,
548  )
549  normalizeGains = pexConfig.Field(
550  dtype=bool,
551  doc="Normalize all the amplifiers in each CCD to have the same median value.",
552  default=False,
553  )
554 
555  # Fringe correction.
556  doFringe = pexConfig.Field(
557  dtype=bool,
558  doc="Apply fringe correction?",
559  default=True,
560  )
561  fringe = pexConfig.ConfigurableField(
562  target=FringeTask,
563  doc="Fringe subtraction task",
564  )
565  fringeAfterFlat = pexConfig.Field(
566  dtype=bool,
567  doc="Do fringe subtraction after flat-fielding?",
568  default=True,
569  )
570 
571  # NAN pixel interpolation option.
572  doNanInterpAfterFlat = pexConfig.Field(
573  dtype=bool,
574  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
575  "also have to interpolate them before flat-fielding."),
576  default=False,
577  )
578 
579  # Distortion model application.
580  doAddDistortionModel = pexConfig.Field(
581  dtype=bool,
582  doc="Apply a distortion model based on camera geometry to the WCS?",
583  default=True,
584  )
585 
586  # Initial CCD-level background statistics options.
587  doMeasureBackground = pexConfig.Field(
588  dtype=bool,
589  doc="Measure the background level on the reduced image?",
590  default=False,
591  )
592 
593  # Camera-specific masking configuration.
594  doCameraSpecificMasking = pexConfig.Field(
595  dtype=bool,
596  doc="Mask camera-specific bad regions?",
597  default=False,
598  )
599  masking = pexConfig.ConfigurableField(
600  target=MaskingTask,
601  doc="Masking task."
602  )
603 
604  # Default photometric calibration options.
605  fluxMag0T1 = pexConfig.DictField(
606  keytype=str,
607  itemtype=float,
608  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
610  ))
611  )
612  defaultFluxMag0T1 = pexConfig.Field(
613  dtype=float,
614  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
615  default=pow(10.0, 0.4*28.0)
616  )
617 
618  # Vignette correction configuration.
619  doVignette = pexConfig.Field(
620  dtype=bool,
621  doc="Apply vignetting parameters?",
622  default=False,
623  )
624  vignette = pexConfig.ConfigurableField(
625  target=VignetteTask,
626  doc="Vignetting task.",
627  )
628 
629  # Transmission curve configuration.
630  doAttachTransmissionCurve = pexConfig.Field(
631  dtype=bool,
632  default=False,
633  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
634  )
635  doUseOpticsTransmission = pexConfig.Field(
636  dtype=bool,
637  default=True,
638  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
639  )
640  doUseFilterTransmission = pexConfig.Field(
641  dtype=bool,
642  default=True,
643  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
644  )
645  doUseSensorTransmission = pexConfig.Field(
646  dtype=bool,
647  default=True,
648  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
649  )
650  doUseAtmosphereTransmission = pexConfig.Field(
651  dtype=bool,
652  default=True,
653  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
654  )
655 
656  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
657  doWrite = pexConfig.Field(
658  dtype=bool,
659  doc="Persist postISRCCD?",
660  default=True,
661  )
662 
663  def validate(self):
664  super().validate()
665  if self.doFlat and self.doApplyGains:
666  raise ValueError("You may not specify both doFlat and doApplyGains")
667 
668 
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670  r"""Apply common instrument signature correction algorithms to a raw frame.
671 
672  The process for correcting imaging data is very similar from
673  camera to camera. This task provides a vanilla implementation of
674  doing these corrections, including the ability to turn certain
675  corrections off if they are not needed. The inputs to the primary
676  method, `run()`, are a raw exposure to be corrected and the
677  calibration data products. The raw input is a single chip sized
678  mosaic of all amps including overscans and other non-science
679  pixels. The method `runDataRef()` identifies and defines the
680  calibration data products, and is intended for use by a
681  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
682  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
683  subclassed for different camera, although the most camera specific
684  methods have been split into subtasks that can be redirected
685  appropriately.
686 
687  The __init__ method sets up the subtasks for ISR processing, using
688  the defaults from `lsst.ip.isr`.
689 
690  Parameters
691  ----------
692  args : `list`
693  Positional arguments passed to the Task constructor. None used at this time.
694  kwargs : `dict`, optional
695  Keyword arguments passed on to the Task constructor. None used at this time.
696  """
697  ConfigClass = IsrTaskConfig
698  _DefaultName = "isr"
699 
700  def __init__(self, **kwargs):
701  super().__init__(**kwargs)
702  self.makeSubtask("assembleCcd")
703  self.makeSubtask("crosstalk")
704  self.makeSubtask("strayLight")
705  self.makeSubtask("fringe")
706  self.makeSubtask("masking")
707  self.makeSubtask("vignette")
708 
709  @classmethod
710  def getInputDatasetTypes(cls, config):
711  inputTypeDict = super().getInputDatasetTypes(config)
712 
713  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
714  # need because the configuration tells us we will not be bothering with the
715  # correction that uses that IDT.
716  if config.doBias is not True:
717  inputTypeDict.pop("bias", None)
718  if config.doLinearize is not True:
719  inputTypeDict.pop("linearizer", None)
720  if config.doCrosstalk is not True:
721  inputTypeDict.pop("crosstalkSources", None)
722  if config.doBrighterFatter is not True:
723  inputTypeDict.pop("bfKernel", None)
724  if config.doDefect is not True:
725  inputTypeDict.pop("defects", None)
726  if config.doDark is not True:
727  inputTypeDict.pop("dark", None)
728  if config.doFlat is not True:
729  inputTypeDict.pop("flat", None)
730  if config.doAttachTransmissionCurve is not True:
731  inputTypeDict.pop("opticsTransmission", None)
732  inputTypeDict.pop("filterTransmission", None)
733  inputTypeDict.pop("sensorTransmission", None)
734  inputTypeDict.pop("atmosphereTransmission", None)
735  if config.doUseOpticsTransmission is not True:
736  inputTypeDict.pop("opticsTransmission", None)
737  if config.doUseFilterTransmission is not True:
738  inputTypeDict.pop("filterTransmission", None)
739  if config.doUseSensorTransmission is not True:
740  inputTypeDict.pop("sensorTransmission", None)
741  if config.doUseAtmosphereTransmission is not True:
742  inputTypeDict.pop("atmosphereTransmission", None)
743 
744  return inputTypeDict
745 
746  @classmethod
747  def getOutputDatasetTypes(cls, config):
748  outputTypeDict = super().getOutputDatasetTypes(config)
749 
750  if config.qa.doThumbnailOss is not True:
751  outputTypeDict.pop("outputOssThumbnail", None)
752  if config.qa.doThumbnailFlattened is not True:
753  outputTypeDict.pop("outputFlattenedThumbnail", None)
754  if config.doWrite is not True:
755  outputTypeDict.pop("outputExposure", None)
756 
757  return outputTypeDict
758 
759  @classmethod
760  def getPrerequisiteDatasetTypes(cls, config):
761  # Input calibration datasets should not constrain the QuantumGraph
762  # (it'd be confusing if not having flats just silently resulted in no
763  # data being processed). Our nomenclature for that is that these are
764  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
765  names = set(cls.getInputDatasetTypes(config))
766  names.remove("ccdExposure")
767  return names
768 
769  @classmethod
770  def getPerDatasetTypeDimensions(cls, config):
771  # Input calibration datasets of different types (i.e. flat, bias) need
772  # not have the same validity range. That makes calibration_label
773  # (which maps directly to a validity range) a "per-DatasetType
774  # dimension".
775  return frozenset(["calibration_label"])
776 
777  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
778  try:
779  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
780  except Exception as e:
781  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
782 
783  inputData['isGen3'] = True
784 
785  if self.config.doLinearize is True:
786  if 'linearizer' not in inputData.keys():
787  detector = inputData['camera'][inputData['detectorNum']]
788  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
789  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
790 
791  if inputData['defects'] is not None:
792  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
793  # masking expects a list of defects defined by their bounding box
794  if not isinstance(inputData["defects"], Defects):
795  inputData["defects"] = Defects.fromTable(inputData["defects"])
796 
797  # Broken: DM-17169
798  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
799  # fixed for non-HSC cameras in the future.
800  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
801  # if self.config.doCrosstalk else None)
802 
803  # Broken: DM-17152
804  # Fringes are not tested to be handled correctly by Gen3 butler.
805  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
806  # assembler=self.assembleCcd
807  # if self.config.doAssembleIsrExposures else None)
808  # if self.config.doFringe and
809  # self.fringe.checkFilter(inputData['ccdExposure'])
810  # else pipeBase.Struct(fringes=None))
811 
812  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
813 
814  def makeDatasetType(self, dsConfig):
815  return super().makeDatasetType(dsConfig)
816 
817  def readIsrData(self, dataRef, rawExposure):
818  """!Retrieve necessary frames for instrument signature removal.
819 
820  Pre-fetching all required ISR data products limits the IO
821  required by the ISR. Any conflict between the calibration data
822  available and that needed for ISR is also detected prior to
823  doing processing, allowing it to fail quickly.
824 
825  Parameters
826  ----------
827  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
828  Butler reference of the detector data to be processed
829  rawExposure : `afw.image.Exposure`
830  The raw exposure that will later be corrected with the
831  retrieved calibration data; should not be modified in this
832  method.
833 
834  Returns
835  -------
836  result : `lsst.pipe.base.Struct`
837  Result struct with components (which may be `None`):
838  - ``bias``: bias calibration frame (`afw.image.Exposure`)
839  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
840  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
841  - ``dark``: dark calibration frame (`afw.image.Exposure`)
842  - ``flat``: flat calibration frame (`afw.image.Exposure`)
843  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
844  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
845  - ``fringes``: `lsst.pipe.base.Struct` with components:
846  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
847  - ``seed``: random seed derived from the ccdExposureId for random
848  number generator (`uint32`)
849  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
850  A ``TransmissionCurve`` that represents the throughput of the optics,
851  to be evaluated in focal-plane coordinates.
852  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
853  A ``TransmissionCurve`` that represents the throughput of the filter
854  itself, to be evaluated in focal-plane coordinates.
855  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
856  A ``TransmissionCurve`` that represents the throughput of the sensor
857  itself, to be evaluated in post-assembly trimmed detector coordinates.
858  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
859  A ``TransmissionCurve`` that represents the throughput of the
860  atmosphere, assumed to be spatially constant.
861  - ``strayLightData`` : `object`
862  An opaque object containing calibration information for
863  stray-light correction. If `None`, no correction will be
864  performed.
865 
866  Raises
867  ------
868  NotImplementedError :
869  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
870  """
871  ccd = rawExposure.getDetector()
872  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
873  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
874  if self.config.doBias else None)
875  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
876  linearizer = (dataRef.get("linearizer", immediate=True)
877  if self.doLinearize(ccd) else None)
878  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
879  if self.config.doCrosstalk else None)
880  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
881  if self.config.doDark else None)
882  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
883  if self.config.doFlat else None)
884 
885  brighterFatterKernel = None
886  if self.config.doBrighterFatter is True:
887 
888  # Use the new-style cp_pipe version of the kernel is it exists.
889  try:
890  brighterFatterKernel = dataRef.get("brighterFatterKernel")
891  except NoResults:
892  # Fall back to the old-style numpy-ndarray style kernel if necessary.
893  try:
894  brighterFatterKernel = dataRef.get("bfKernel")
895  except NoResults:
896  brighterFatterKernel = None
897  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
898  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
899  # this detector, or raise an error.
900  if self.config.brighterFatterLevel == 'DETECTOR':
901  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
902  else:
903  # TODO DM-15631 for implementing this
904  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
905 
906  defectList = (dataRef.get("defects")
907  if self.config.doDefect else None)
908  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
909  if self.config.doAssembleIsrExposures else None)
910  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
911  else pipeBase.Struct(fringes=None))
912 
913  if self.config.doAttachTransmissionCurve:
914  opticsTransmission = (dataRef.get("transmission_optics")
915  if self.config.doUseOpticsTransmission else None)
916  filterTransmission = (dataRef.get("transmission_filter")
917  if self.config.doUseFilterTransmission else None)
918  sensorTransmission = (dataRef.get("transmission_sensor")
919  if self.config.doUseSensorTransmission else None)
920  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
921  if self.config.doUseAtmosphereTransmission else None)
922  else:
923  opticsTransmission = None
924  filterTransmission = None
925  sensorTransmission = None
926  atmosphereTransmission = None
927 
928  if self.config.doStrayLight:
929  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
930  else:
931  strayLightData = None
932 
933  # Struct should include only kwargs to run()
934  return pipeBase.Struct(bias=biasExposure,
935  linearizer=linearizer,
936  crosstalkSources=crosstalkSources,
937  dark=darkExposure,
938  flat=flatExposure,
939  bfKernel=brighterFatterKernel,
940  defects=defectList,
941  fringes=fringeStruct,
942  opticsTransmission=opticsTransmission,
943  filterTransmission=filterTransmission,
944  sensorTransmission=sensorTransmission,
945  atmosphereTransmission=atmosphereTransmission,
946  strayLightData=strayLightData
947  )
948 
949  @pipeBase.timeMethod
950  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
951  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
952  opticsTransmission=None, filterTransmission=None,
953  sensorTransmission=None, atmosphereTransmission=None,
954  detectorNum=None, strayLightData=None, isGen3=False,
955  ):
956  """!Perform instrument signature removal on an exposure.
957 
958  Steps included in the ISR processing, in order performed, are:
959  - saturation and suspect pixel masking
960  - overscan subtraction
961  - CCD assembly of individual amplifiers
962  - bias subtraction
963  - variance image construction
964  - linearization of non-linear response
965  - crosstalk masking
966  - brighter-fatter correction
967  - dark subtraction
968  - fringe correction
969  - stray light subtraction
970  - flat correction
971  - masking of known defects and camera specific features
972  - vignette calculation
973  - appending transmission curve and distortion model
974 
975  Parameters
976  ----------
977  ccdExposure : `lsst.afw.image.Exposure`
978  The raw exposure that is to be run through ISR. The
979  exposure is modified by this method.
980  camera : `lsst.afw.cameraGeom.Camera`, optional
981  The camera geometry for this exposure. Used to select the
982  distortion model appropriate for this data.
983  bias : `lsst.afw.image.Exposure`, optional
984  Bias calibration frame.
985  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
986  Functor for linearization.
987  crosstalkSources : `list`, optional
988  List of possible crosstalk sources.
989  dark : `lsst.afw.image.Exposure`, optional
990  Dark calibration frame.
991  flat : `lsst.afw.image.Exposure`, optional
992  Flat calibration frame.
993  bfKernel : `numpy.ndarray`, optional
994  Brighter-fatter kernel.
995  defects : `lsst.meas.algorithms.Defects`, optional
996  List of defects.
997  fringes : `lsst.pipe.base.Struct`, optional
998  Struct containing the fringe correction data, with
999  elements:
1000  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1001  - ``seed``: random seed derived from the ccdExposureId for random
1002  number generator (`uint32`)
1003  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1004  A ``TransmissionCurve`` that represents the throughput of the optics,
1005  to be evaluated in focal-plane coordinates.
1006  filterTransmission : `lsst.afw.image.TransmissionCurve`
1007  A ``TransmissionCurve`` that represents the throughput of the filter
1008  itself, to be evaluated in focal-plane coordinates.
1009  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1010  A ``TransmissionCurve`` that represents the throughput of the sensor
1011  itself, to be evaluated in post-assembly trimmed detector coordinates.
1012  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1013  A ``TransmissionCurve`` that represents the throughput of the
1014  atmosphere, assumed to be spatially constant.
1015  detectorNum : `int`, optional
1016  The integer number for the detector to process.
1017  isGen3 : bool, optional
1018  Flag this call to run() as using the Gen3 butler environment.
1019  strayLightData : `object`, optional
1020  Opaque object containing calibration information for stray-light
1021  correction. If `None`, no correction will be performed.
1022 
1023  Returns
1024  -------
1025  result : `lsst.pipe.base.Struct`
1026  Result struct with component:
1027  - ``exposure`` : `afw.image.Exposure`
1028  The fully ISR corrected exposure.
1029  - ``outputExposure`` : `afw.image.Exposure`
1030  An alias for `exposure`
1031  - ``ossThumb`` : `numpy.ndarray`
1032  Thumbnail image of the exposure after overscan subtraction.
1033  - ``flattenedThumb`` : `numpy.ndarray`
1034  Thumbnail image of the exposure after flat-field correction.
1035 
1036  Raises
1037  ------
1038  RuntimeError
1039  Raised if a configuration option is set to True, but the
1040  required calibration data has not been specified.
1041 
1042  Notes
1043  -----
1044  The current processed exposure can be viewed by setting the
1045  appropriate lsstDebug entries in the `debug.display`
1046  dictionary. The names of these entries correspond to some of
1047  the IsrTaskConfig Boolean options, with the value denoting the
1048  frame to use. The exposure is shown inside the matching
1049  option check and after the processing of that step has
1050  finished. The steps with debug points are:
1051 
1052  doAssembleCcd
1053  doBias
1054  doCrosstalk
1055  doBrighterFatter
1056  doDark
1057  doFringe
1058  doStrayLight
1059  doFlat
1060 
1061  In addition, setting the "postISRCCD" entry displays the
1062  exposure after all ISR processing has finished.
1063 
1064  """
1065 
1066  if isGen3 is True:
1067  # Gen3 currently cannot automatically do configuration overrides.
1068  # DM-15257 looks to discuss this issue.
1069 
1070  self.config.doFringe = False
1071 
1072  # Configure input exposures;
1073  if detectorNum is None:
1074  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1075 
1076  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1077  bias = self.ensureExposure(bias, camera, detectorNum)
1078  dark = self.ensureExposure(dark, camera, detectorNum)
1079  flat = self.ensureExposure(flat, camera, detectorNum)
1080  else:
1081  if isinstance(ccdExposure, ButlerDataRef):
1082  return self.runDataRef(ccdExposure)
1083 
1084  ccd = ccdExposure.getDetector()
1085 
1086  if not ccd:
1087  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1088  ccd = [FakeAmp(ccdExposure, self.config)]
1089 
1090  # Validate Input
1091  if self.config.doBias and bias is None:
1092  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1093  if self.doLinearize(ccd) and linearizer is None:
1094  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1095  if self.config.doBrighterFatter and bfKernel is None:
1096  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1097  if self.config.doDark and dark is None:
1098  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1099  if fringes is None:
1100  fringes = pipeBase.Struct(fringes=None)
1101  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1102  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1103  if self.config.doFlat and flat is None:
1104  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1105  if self.config.doDefect and defects is None:
1106  raise RuntimeError("Must supply defects if config.doDefect=True.")
1107  if self.config.doAddDistortionModel and camera is None:
1108  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1109 
1110  # Begin ISR processing.
1111  if self.config.doConvertIntToFloat:
1112  self.log.info("Converting exposure to floating point values")
1113  ccdExposure = self.convertIntToFloat(ccdExposure)
1114 
1115  # Amplifier level processing.
1116  overscans = []
1117  for amp in ccd:
1118  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1119  if ccdExposure.getBBox().contains(amp.getBBox()):
1120  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1121  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1122 
1123  if self.config.doOverscan and not badAmp:
1124  # Overscan correction on amp-by-amp basis.
1125  overscanResults = self.overscanCorrection(ccdExposure, amp)
1126  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1127  if self.config.qa is not None and self.config.qa.saveStats is True:
1128  if isinstance(overscanResults.overscanFit, float):
1129  qaMedian = overscanResults.overscanFit
1130  qaStdev = float("NaN")
1131  else:
1132  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1133  afwMath.MEDIAN | afwMath.STDEVCLIP)
1134  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1135  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1136 
1137  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1138  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1139  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1140  (amp.getName(), qaMedian, qaStdev))
1141  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1142  else:
1143  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1144  overscanResults = None
1145 
1146  overscans.append(overscanResults if overscanResults is not None else None)
1147  else:
1148  self.log.info("Skipped OSCAN")
1149 
1150  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1151  self.log.info("Applying crosstalk correction.")
1152  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1153  self.debugView(ccdExposure, "doCrosstalk")
1154 
1155  if self.config.doAssembleCcd:
1156  self.log.info("Assembling CCD from amplifiers")
1157  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1158 
1159  if self.config.expectWcs and not ccdExposure.getWcs():
1160  self.log.warn("No WCS found in input exposure")
1161  self.debugView(ccdExposure, "doAssembleCcd")
1162 
1163  ossThumb = None
1164  if self.config.qa.doThumbnailOss:
1165  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1166 
1167  if self.config.doBias:
1168  self.log.info("Applying bias correction.")
1169  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1170  trimToFit=self.config.doTrimToMatchCalib)
1171  self.debugView(ccdExposure, "doBias")
1172 
1173  if self.config.doVariance:
1174  for amp, overscanResults in zip(ccd, overscans):
1175  if ccdExposure.getBBox().contains(amp.getBBox()):
1176  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1177  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1178  if overscanResults is not None:
1179  self.updateVariance(ampExposure, amp,
1180  overscanImage=overscanResults.overscanImage)
1181  else:
1182  self.updateVariance(ampExposure, amp,
1183  overscanImage=None)
1184  if self.config.qa is not None and self.config.qa.saveStats is True:
1185  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1186  afwMath.MEDIAN | afwMath.STDEVCLIP)
1187  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1188  qaStats.getValue(afwMath.MEDIAN))
1189  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1190  qaStats.getValue(afwMath.STDEVCLIP))
1191  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1192  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1193  qaStats.getValue(afwMath.STDEVCLIP)))
1194 
1195  if self.doLinearize(ccd):
1196  self.log.info("Applying linearizer.")
1197  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1198 
1199  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1200  self.log.info("Applying crosstalk correction.")
1201  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1202  self.debugView(ccdExposure, "doCrosstalk")
1203 
1204  if self.config.doWidenSaturationTrails:
1205  self.log.info("Widening saturation trails.")
1206  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1207 
1208  interpolationDone = False
1209  if self.config.doBrighterFatter:
1210  # We need to apply flats and darks before we can interpolate, and we
1211  # need to interpolate before we do B-F, but we do B-F without the
1212  # flats and darks applied so we can work in units of electrons or holes.
1213  # This context manager applies and then removes the darks and flats.
1214  with self.flatContext(ccdExposure, flat, dark):
1215  if self.config.doDefect:
1216  self.maskAndInterpDefect(ccdExposure, defects)
1217 
1218  if self.config.doSaturationInterpolation:
1219  self.saturationInterpolation(ccdExposure)
1220 
1221  self.maskAndInterpNan(ccdExposure)
1222  interpolationDone = True
1223 
1224  self.log.info("Applying brighter fatter correction.")
1225  isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1226  self.config.brighterFatterMaxIter,
1227  self.config.brighterFatterThreshold,
1228  self.config.brighterFatterApplyGain,
1229  )
1230  self.debugView(ccdExposure, "doBrighterFatter")
1231 
1232  if self.config.doDark:
1233  self.log.info("Applying dark correction.")
1234  self.darkCorrection(ccdExposure, dark)
1235  self.debugView(ccdExposure, "doDark")
1236 
1237  if self.config.doFringe and not self.config.fringeAfterFlat:
1238  self.log.info("Applying fringe correction before flat.")
1239  self.fringe.run(ccdExposure, **fringes.getDict())
1240  self.debugView(ccdExposure, "doFringe")
1241 
1242  if self.config.doStrayLight:
1243  if strayLightData is not None:
1244  self.log.info("Applying stray light correction.")
1245  self.strayLight.run(ccdExposure, strayLightData)
1246  self.debugView(ccdExposure, "doStrayLight")
1247  else:
1248  self.log.debug("Skipping stray light correction: no data found for this image.")
1249 
1250  if self.config.doFlat:
1251  self.log.info("Applying flat correction.")
1252  self.flatCorrection(ccdExposure, flat)
1253  self.debugView(ccdExposure, "doFlat")
1254 
1255  if self.config.doApplyGains:
1256  self.log.info("Applying gain correction instead of flat.")
1257  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1258 
1259  if self.config.doDefect and not interpolationDone:
1260  self.log.info("Masking and interpolating defects.")
1261  self.maskAndInterpDefect(ccdExposure, defects)
1262 
1263  if self.config.doSaturationInterpolation and not interpolationDone:
1264  self.log.info("Interpolating saturated pixels.")
1265  self.saturationInterpolation(ccdExposure)
1266 
1267  if self.config.doNanInterpAfterFlat or not interpolationDone:
1268  self.log.info("Masking and interpolating NAN value pixels.")
1269  self.maskAndInterpNan(ccdExposure)
1270 
1271  if self.config.doFringe and self.config.fringeAfterFlat:
1272  self.log.info("Applying fringe correction after flat.")
1273  self.fringe.run(ccdExposure, **fringes.getDict())
1274 
1275  if self.config.doSetBadRegions:
1276  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1277  if badPixelCount > 0:
1278  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1279 
1280  flattenedThumb = None
1281  if self.config.qa.doThumbnailFlattened:
1282  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1283 
1284  if self.config.doCameraSpecificMasking:
1285  self.log.info("Masking regions for camera specific reasons.")
1286  self.masking.run(ccdExposure)
1287 
1288  self.roughZeroPoint(ccdExposure)
1289 
1290  if self.config.doVignette:
1291  self.log.info("Constructing Vignette polygon.")
1292  self.vignettePolygon = self.vignette.run(ccdExposure)
1293 
1294  if self.config.vignette.doWriteVignettePolygon:
1295  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1296 
1297  if self.config.doAttachTransmissionCurve:
1298  self.log.info("Adding transmission curves.")
1299  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1300  filterTransmission=filterTransmission,
1301  sensorTransmission=sensorTransmission,
1302  atmosphereTransmission=atmosphereTransmission)
1303 
1304  if self.config.doAddDistortionModel:
1305  self.log.info("Adding a distortion model to the WCS.")
1306  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1307 
1308  if self.config.doMeasureBackground:
1309  self.log.info("Measuring background level:")
1310  self.measureBackground(ccdExposure, self.config.qa)
1311 
1312  if self.config.qa is not None and self.config.qa.saveStats is True:
1313  for amp in ccd:
1314  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1315  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1316  afwMath.MEDIAN | afwMath.STDEVCLIP)
1317  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1318  qaStats.getValue(afwMath.MEDIAN))
1319  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1320  qaStats.getValue(afwMath.STDEVCLIP))
1321  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1322  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1323  qaStats.getValue(afwMath.STDEVCLIP)))
1324 
1325  self.debugView(ccdExposure, "postISRCCD")
1326 
1327  return pipeBase.Struct(
1328  exposure=ccdExposure,
1329  ossThumb=ossThumb,
1330  flattenedThumb=flattenedThumb,
1331 
1332  outputExposure=ccdExposure,
1333  outputOssThumbnail=ossThumb,
1334  outputFlattenedThumbnail=flattenedThumb,
1335  )
1336 
1337  @pipeBase.timeMethod
1338  def runDataRef(self, sensorRef):
1339  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1340 
1341  This method contains the `CmdLineTask` interface to the ISR
1342  processing. All IO is handled here, freeing the `run()` method
1343  to manage only pixel-level calculations. The steps performed
1344  are:
1345  - Read in necessary detrending/isr/calibration data.
1346  - Process raw exposure in `run()`.
1347  - Persist the ISR-corrected exposure as "postISRCCD" if
1348  config.doWrite=True.
1349 
1350  Parameters
1351  ----------
1352  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1353  DataRef of the detector data to be processed
1354 
1355  Returns
1356  -------
1357  result : `lsst.pipe.base.Struct`
1358  Result struct with component:
1359  - ``exposure`` : `afw.image.Exposure`
1360  The fully ISR corrected exposure.
1361 
1362  Raises
1363  ------
1364  RuntimeError
1365  Raised if a configuration option is set to True, but the
1366  required calibration data does not exist.
1367 
1368  """
1369  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1370 
1371  ccdExposure = sensorRef.get(self.config.datasetType)
1372 
1373  camera = sensorRef.get("camera")
1374  if camera is None and self.config.doAddDistortionModel:
1375  raise RuntimeError("config.doAddDistortionModel is True "
1376  "but could not get a camera from the butler")
1377  isrData = self.readIsrData(sensorRef, ccdExposure)
1378 
1379  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1380 
1381  if self.config.doWrite:
1382  sensorRef.put(result.exposure, "postISRCCD")
1383  if result.ossThumb is not None:
1384  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1385  if result.flattenedThumb is not None:
1386  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1387 
1388  return result
1389 
1390  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1391  """!Retrieve a calibration dataset for removing instrument signature.
1392 
1393  Parameters
1394  ----------
1395 
1396  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1397  DataRef of the detector data to find calibration datasets
1398  for.
1399  datasetType : `str`
1400  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1401  immediate : `Bool`
1402  If True, disable butler proxies to enable error handling
1403  within this routine.
1404 
1405  Returns
1406  -------
1407  exposure : `lsst.afw.image.Exposure`
1408  Requested calibration frame.
1409 
1410  Raises
1411  ------
1412  RuntimeError
1413  Raised if no matching calibration frame can be found.
1414  """
1415  try:
1416  exp = dataRef.get(datasetType, immediate=immediate)
1417  except Exception as exc1:
1418  if not self.config.fallbackFilterName:
1419  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1420  try:
1421  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1422  except Exception as exc2:
1423  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1424  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1425  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1426 
1427  if self.config.doAssembleIsrExposures:
1428  exp = self.assembleCcd.assembleCcd(exp)
1429  return exp
1430 
1431  def ensureExposure(self, inputExp, camera, detectorNum):
1432  """Ensure that the data returned by Butler is a fully constructed exposure.
1433 
1434  ISR requires exposure-level image data for historical reasons, so if we did
1435  not recieve that from Butler, construct it from what we have, modifying the
1436  input in place.
1437 
1438  Parameters
1439  ----------
1440  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1441  `lsst.afw.image.ImageF`
1442  The input data structure obtained from Butler.
1443  camera : `lsst.afw.cameraGeom.camera`
1444  The camera associated with the image. Used to find the appropriate
1445  detector.
1446  detectorNum : `int`
1447  The detector this exposure should match.
1448 
1449  Returns
1450  -------
1451  inputExp : `lsst.afw.image.Exposure`
1452  The re-constructed exposure, with appropriate detector parameters.
1453 
1454  Raises
1455  ------
1456  TypeError
1457  Raised if the input data cannot be used to construct an exposure.
1458  """
1459  if isinstance(inputExp, afwImage.DecoratedImageU):
1460  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1461  elif isinstance(inputExp, afwImage.ImageF):
1462  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1463  elif isinstance(inputExp, afwImage.MaskedImageF):
1464  inputExp = afwImage.makeExposure(inputExp)
1465  elif isinstance(inputExp, afwImage.Exposure):
1466  pass
1467  else:
1468  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1469 
1470  if inputExp.getDetector() is None:
1471  inputExp.setDetector(camera[detectorNum])
1472 
1473  return inputExp
1474 
1475  def convertIntToFloat(self, exposure):
1476  """Convert exposure image from uint16 to float.
1477 
1478  If the exposure does not need to be converted, the input is
1479  immediately returned. For exposures that are converted to use
1480  floating point pixels, the variance is set to unity and the
1481  mask to zero.
1482 
1483  Parameters
1484  ----------
1485  exposure : `lsst.afw.image.Exposure`
1486  The raw exposure to be converted.
1487 
1488  Returns
1489  -------
1490  newexposure : `lsst.afw.image.Exposure`
1491  The input ``exposure``, converted to floating point pixels.
1492 
1493  Raises
1494  ------
1495  RuntimeError
1496  Raised if the exposure type cannot be converted to float.
1497 
1498  """
1499  if isinstance(exposure, afwImage.ExposureF):
1500  # Nothing to be done
1501  return exposure
1502  if not hasattr(exposure, "convertF"):
1503  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1504 
1505  newexposure = exposure.convertF()
1506  newexposure.variance[:] = 1
1507  newexposure.mask[:] = 0x0
1508 
1509  return newexposure
1510 
1511  def maskAmplifier(self, ccdExposure, amp, defects):
1512  """Identify bad amplifiers, saturated and suspect pixels.
1513 
1514  Parameters
1515  ----------
1516  ccdExposure : `lsst.afw.image.Exposure`
1517  Input exposure to be masked.
1518  amp : `lsst.afw.table.AmpInfoCatalog`
1519  Catalog of parameters defining the amplifier on this
1520  exposure to mask.
1521  defects : `lsst.meas.algorithms.Defects`
1522  List of defects. Used to determine if the entire
1523  amplifier is bad.
1524 
1525  Returns
1526  -------
1527  badAmp : `Bool`
1528  If this is true, the entire amplifier area is covered by
1529  defects and unusable.
1530 
1531  """
1532  maskedImage = ccdExposure.getMaskedImage()
1533 
1534  badAmp = False
1535 
1536  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1537  # comparison with current defects definition.
1538  if defects is not None:
1539  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1540 
1541  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1542  # association with pixels in current ccdExposure).
1543  if badAmp:
1544  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1545  afwImage.PARENT)
1546  maskView = dataView.getMask()
1547  maskView |= maskView.getPlaneBitMask("BAD")
1548  del maskView
1549  return badAmp
1550 
1551  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1552  # Saturation and suspect pixels can be masked now, though.
1553  limits = dict()
1554  if self.config.doSaturation and not badAmp:
1555  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1556  if self.config.doSuspect and not badAmp:
1557  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1558 
1559  for maskName, maskThreshold in limits.items():
1560  if not math.isnan(maskThreshold):
1561  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1562  isrFunctions.makeThresholdMask(
1563  maskedImage=dataView,
1564  threshold=maskThreshold,
1565  growFootprints=0,
1566  maskName=maskName
1567  )
1568 
1569  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1570  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1571  afwImage.PARENT)
1572  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1573  self.config.suspectMaskName])
1574  if numpy.all(maskView.getArray() & maskVal > 0):
1575  badAmp = True
1576 
1577  return badAmp
1578 
1579  def overscanCorrection(self, ccdExposure, amp):
1580  """Apply overscan correction in place.
1581 
1582  This method does initial pixel rejection of the overscan
1583  region. The overscan can also be optionally segmented to
1584  allow for discontinuous overscan responses to be fit
1585  separately. The actual overscan subtraction is performed by
1586  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1587  which is called here after the amplifier is preprocessed.
1588 
1589  Parameters
1590  ----------
1591  ccdExposure : `lsst.afw.image.Exposure`
1592  Exposure to have overscan correction performed.
1593  amp : `lsst.afw.table.AmpInfoCatalog`
1594  The amplifier to consider while correcting the overscan.
1595 
1596  Returns
1597  -------
1598  overscanResults : `lsst.pipe.base.Struct`
1599  Result struct with components:
1600  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1601  Value or fit subtracted from the amplifier image data.
1602  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1603  Value or fit subtracted from the overscan image data.
1604  - ``overscanImage`` : `lsst.afw.image.Image`
1605  Image of the overscan region with the overscan
1606  correction applied. This quantity is used to estimate
1607  the amplifier read noise empirically.
1608 
1609  Raises
1610  ------
1611  RuntimeError
1612  Raised if the ``amp`` does not contain raw pixel information.
1613 
1614  See Also
1615  --------
1616  lsst.ip.isr.isrFunctions.overscanCorrection
1617  """
1618  if not amp.getHasRawInfo():
1619  raise RuntimeError("This method must be executed on an amp with raw information.")
1620 
1621  if amp.getRawHorizontalOverscanBBox().isEmpty():
1622  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1623  return None
1624 
1625  statControl = afwMath.StatisticsControl()
1626  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1627 
1628  # Determine the bounding boxes
1629  dataBBox = amp.getRawDataBBox()
1630  oscanBBox = amp.getRawHorizontalOverscanBBox()
1631  dx0 = 0
1632  dx1 = 0
1633 
1634  prescanBBox = amp.getRawPrescanBBox()
1635  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1636  dx0 += self.config.overscanNumLeadingColumnsToSkip
1637  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1638  else:
1639  dx0 += self.config.overscanNumTrailingColumnsToSkip
1640  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1641 
1642  # Determine if we need to work on subregions of the amplifier and overscan.
1643  imageBBoxes = []
1644  overscanBBoxes = []
1645 
1646  if ((self.config.overscanBiasJump and
1647  self.config.overscanBiasJumpLocation) and
1648  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1649  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1650  self.config.overscanBiasJumpDevices)):
1651  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1652  yLower = self.config.overscanBiasJumpLocation
1653  yUpper = dataBBox.getHeight() - yLower
1654  else:
1655  yUpper = self.config.overscanBiasJumpLocation
1656  yLower = dataBBox.getHeight() - yUpper
1657 
1658  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1659  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1660  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1661  lsst.geom.Extent2I(dx0, 0),
1662  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1663  yLower)))
1664 
1665  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1666  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1667  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1668  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1669  yUpper)))
1670  else:
1671  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1672  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1673  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1674  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1675  oscanBBox.getHeight())))
1676 
1677  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1678  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1679  ampImage = ccdExposure.maskedImage[imageBBox]
1680  overscanImage = ccdExposure.maskedImage[overscanBBox]
1681 
1682  overscanArray = overscanImage.image.array
1683  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1684  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1685  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1686 
1687  statControl = afwMath.StatisticsControl()
1688  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1689 
1690  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1691  overscanImage=overscanImage,
1692  fitType=self.config.overscanFitType,
1693  order=self.config.overscanOrder,
1694  collapseRej=self.config.overscanNumSigmaClip,
1695  statControl=statControl,
1696  overscanIsInt=self.config.overscanIsInt
1697  )
1698 
1699  # Measure average overscan levels and record them in the metadata
1700  levelStat = afwMath.MEDIAN
1701  sigmaStat = afwMath.STDEVCLIP
1702 
1703  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1704  self.config.qa.flatness.nIter)
1705  metadata = ccdExposure.getMetadata()
1706  ampNum = amp.getName()
1707  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1708  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1709  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1710  else:
1711  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1712  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1713  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1714 
1715  return overscanResults
1716 
1717  def updateVariance(self, ampExposure, amp, overscanImage=None):
1718  """Set the variance plane using the amplifier gain and read noise
1719 
1720  The read noise is calculated from the ``overscanImage`` if the
1721  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1722  the value from the amplifier data is used.
1723 
1724  Parameters
1725  ----------
1726  ampExposure : `lsst.afw.image.Exposure`
1727  Exposure to process.
1728  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1729  Amplifier detector data.
1730  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1731  Image of overscan, required only for empirical read noise.
1732 
1733  See also
1734  --------
1735  lsst.ip.isr.isrFunctions.updateVariance
1736  """
1737  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1738  gain = amp.getGain()
1739 
1740  if math.isnan(gain):
1741  gain = 1.0
1742  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1743  elif gain <= 0:
1744  patchedGain = 1.0
1745  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1746  (amp.getName(), gain, patchedGain))
1747  gain = patchedGain
1748 
1749  if self.config.doEmpiricalReadNoise and overscanImage is None:
1750  self.log.info("Overscan is none for EmpiricalReadNoise")
1751 
1752  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1753  stats = afwMath.StatisticsControl()
1754  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1755  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1756  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1757  else:
1758  readNoise = amp.getReadNoise()
1759 
1760  isrFunctions.updateVariance(
1761  maskedImage=ampExposure.getMaskedImage(),
1762  gain=gain,
1763  readNoise=readNoise,
1764  )
1765 
1766  def darkCorrection(self, exposure, darkExposure, invert=False):
1767  """!Apply dark correction in place.
1768 
1769  Parameters
1770  ----------
1771  exposure : `lsst.afw.image.Exposure`
1772  Exposure to process.
1773  darkExposure : `lsst.afw.image.Exposure`
1774  Dark exposure of the same size as ``exposure``.
1775  invert : `Bool`, optional
1776  If True, re-add the dark to an already corrected image.
1777 
1778  Raises
1779  ------
1780  RuntimeError
1781  Raised if either ``exposure`` or ``darkExposure`` do not
1782  have their dark time defined.
1783 
1784  See Also
1785  --------
1786  lsst.ip.isr.isrFunctions.darkCorrection
1787  """
1788  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1789  if math.isnan(expScale):
1790  raise RuntimeError("Exposure darktime is NAN")
1791  if darkExposure.getInfo().getVisitInfo() is not None:
1792  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1793  else:
1794  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1795  # so getDarkTime() does not exist.
1796  darkScale = 1.0
1797 
1798  if math.isnan(darkScale):
1799  raise RuntimeError("Dark calib darktime is NAN")
1800  isrFunctions.darkCorrection(
1801  maskedImage=exposure.getMaskedImage(),
1802  darkMaskedImage=darkExposure.getMaskedImage(),
1803  expScale=expScale,
1804  darkScale=darkScale,
1805  invert=invert,
1806  trimToFit=self.config.doTrimToMatchCalib
1807  )
1808 
1809  def doLinearize(self, detector):
1810  """!Check if linearization is needed for the detector cameraGeom.
1811 
1812  Checks config.doLinearize and the linearity type of the first
1813  amplifier.
1814 
1815  Parameters
1816  ----------
1817  detector : `lsst.afw.cameraGeom.Detector`
1818  Detector to get linearity type from.
1819 
1820  Returns
1821  -------
1822  doLinearize : `Bool`
1823  If True, linearization should be performed.
1824  """
1825  return self.config.doLinearize and \
1826  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1827 
1828  def flatCorrection(self, exposure, flatExposure, invert=False):
1829  """!Apply flat correction in place.
1830 
1831  Parameters
1832  ----------
1833  exposure : `lsst.afw.image.Exposure`
1834  Exposure to process.
1835  flatExposure : `lsst.afw.image.Exposure`
1836  Flat exposure of the same size as ``exposure``.
1837  invert : `Bool`, optional
1838  If True, unflatten an already flattened image.
1839 
1840  See Also
1841  --------
1842  lsst.ip.isr.isrFunctions.flatCorrection
1843  """
1844  isrFunctions.flatCorrection(
1845  maskedImage=exposure.getMaskedImage(),
1846  flatMaskedImage=flatExposure.getMaskedImage(),
1847  scalingType=self.config.flatScalingType,
1848  userScale=self.config.flatUserScale,
1849  invert=invert,
1850  trimToFit=self.config.doTrimToMatchCalib
1851  )
1852 
1853  def saturationDetection(self, exposure, amp):
1854  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1855 
1856  Parameters
1857  ----------
1858  exposure : `lsst.afw.image.Exposure`
1859  Exposure to process. Only the amplifier DataSec is processed.
1860  amp : `lsst.afw.table.AmpInfoCatalog`
1861  Amplifier detector data.
1862 
1863  See Also
1864  --------
1865  lsst.ip.isr.isrFunctions.makeThresholdMask
1866  """
1867  if not math.isnan(amp.getSaturation()):
1868  maskedImage = exposure.getMaskedImage()
1869  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1870  isrFunctions.makeThresholdMask(
1871  maskedImage=dataView,
1872  threshold=amp.getSaturation(),
1873  growFootprints=0,
1874  maskName=self.config.saturatedMaskName,
1875  )
1876 
1877  def saturationInterpolation(self, ccdExposure):
1878  """!Interpolate over saturated pixels, in place.
1879 
1880  This method should be called after `saturationDetection`, to
1881  ensure that the saturated pixels have been identified in the
1882  SAT mask. It should also be called after `assembleCcd`, since
1883  saturated regions may cross amplifier boundaries.
1884 
1885  Parameters
1886  ----------
1887  exposure : `lsst.afw.image.Exposure`
1888  Exposure to process.
1889 
1890  See Also
1891  --------
1892  lsst.ip.isr.isrTask.saturationDetection
1893  lsst.ip.isr.isrFunctions.interpolateFromMask
1894  """
1895  isrFunctions.interpolateFromMask(
1896  maskedImage=ccdExposure.getMaskedImage(),
1897  fwhm=self.config.fwhm,
1898  growFootprints=self.config.growSaturationFootprintSize,
1899  maskName=self.config.saturatedMaskName,
1900  )
1901 
1902  def suspectDetection(self, exposure, amp):
1903  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1904 
1905  Parameters
1906  ----------
1907  exposure : `lsst.afw.image.Exposure`
1908  Exposure to process. Only the amplifier DataSec is processed.
1909  amp : `lsst.afw.table.AmpInfoCatalog`
1910  Amplifier detector data.
1911 
1912  See Also
1913  --------
1914  lsst.ip.isr.isrFunctions.makeThresholdMask
1915 
1916  Notes
1917  -----
1918  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1919  This is intended to indicate pixels that may be affected by unknown systematics;
1920  for example if non-linearity corrections above a certain level are unstable
1921  then that would be a useful value for suspectLevel. A value of `nan` indicates
1922  that no such level exists and no pixels are to be masked as suspicious.
1923  """
1924  suspectLevel = amp.getSuspectLevel()
1925  if math.isnan(suspectLevel):
1926  return
1927 
1928  maskedImage = exposure.getMaskedImage()
1929  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1930  isrFunctions.makeThresholdMask(
1931  maskedImage=dataView,
1932  threshold=suspectLevel,
1933  growFootprints=0,
1934  maskName=self.config.suspectMaskName,
1935  )
1936 
1937  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1938  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1939 
1940  Parameters
1941  ----------
1942  ccdExposure : `lsst.afw.image.Exposure`
1943  Exposure to process.
1944  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
1945  `lsst.afw.image.DefectBase`.
1946  List of defects to mask and interpolate.
1947 
1948  Notes
1949  -----
1950  Call this after CCD assembly, since defects may cross amplifier boundaries.
1951  """
1952  maskedImage = ccdExposure.getMaskedImage()
1953  if not isinstance(defectBaseList, Defects):
1954  # Promotes DefectBase to Defect
1955  defectList = Defects(defectBaseList)
1956  else:
1957  defectList = defectBaseList
1958  defectList.maskPixels(maskedImage, maskName="BAD")
1959  isrFunctions.interpolateDefectList(
1960  maskedImage=maskedImage,
1961  defectList=defectList,
1962  fwhm=self.config.fwhm,
1963  )
1964 
1965  if self.config.numEdgeSuspect > 0:
1966  goodBBox = maskedImage.getBBox()
1967  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1968  goodBBox.grow(-self.config.numEdgeSuspect)
1969  # Mask pixels outside goodBBox as SUSPECT
1970  SourceDetectionTask.setEdgeBits(
1971  maskedImage,
1972  goodBBox,
1973  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1974  )
1975 
1976  def maskAndInterpNan(self, exposure):
1977  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1978 
1979  Parameters
1980  ----------
1981  exposure : `lsst.afw.image.Exposure`
1982  Exposure to process.
1983 
1984  Notes
1985  -----
1986  We mask and interpolate over all NaNs, including those
1987  that are masked with other bits (because those may or may
1988  not be interpolated over later, and we want to remove all
1989  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1990  is used to preserve the historical name.
1991  """
1992  maskedImage = exposure.getMaskedImage()
1993 
1994  # Find and mask NaNs
1995  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1996  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1997  numNans = maskNans(maskedImage, maskVal)
1998  self.metadata.set("NUMNANS", numNans)
1999 
2000  # Interpolate over these previously-unmasked NaNs
2001  if numNans > 0:
2002  self.log.warn("There were %i unmasked NaNs", numNans)
2003  nanDefectList = Defects.fromMask(
2004  maskedImage=maskedImage,
2005  maskName='UNMASKEDNAN',
2006  )
2007  isrFunctions.interpolateDefectList(
2008  maskedImage=exposure.getMaskedImage(),
2009  defectList=nanDefectList,
2010  fwhm=self.config.fwhm,
2011  )
2012 
2013  def measureBackground(self, exposure, IsrQaConfig=None):
2014  """Measure the image background in subgrids, for quality control purposes.
2015 
2016  Parameters
2017  ----------
2018  exposure : `lsst.afw.image.Exposure`
2019  Exposure to process.
2020  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2021  Configuration object containing parameters on which background
2022  statistics and subgrids to use.
2023  """
2024  if IsrQaConfig is not None:
2025  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2026  IsrQaConfig.flatness.nIter)
2027  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2028  statsControl.setAndMask(maskVal)
2029  maskedImage = exposure.getMaskedImage()
2030  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2031  skyLevel = stats.getValue(afwMath.MEDIAN)
2032  skySigma = stats.getValue(afwMath.STDEVCLIP)
2033  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2034  metadata = exposure.getMetadata()
2035  metadata.set('SKYLEVEL', skyLevel)
2036  metadata.set('SKYSIGMA', skySigma)
2037 
2038  # calcluating flatlevel over the subgrids
2039  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2040  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2041  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2042  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2043  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2044  skyLevels = numpy.zeros((nX, nY))
2045 
2046  for j in range(nY):
2047  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2048  for i in range(nX):
2049  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2050 
2051  xLLC = xc - meshXHalf
2052  yLLC = yc - meshYHalf
2053  xURC = xc + meshXHalf - 1
2054  yURC = yc + meshYHalf - 1
2055 
2056  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2057  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2058 
2059  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2060 
2061  good = numpy.where(numpy.isfinite(skyLevels))
2062  skyMedian = numpy.median(skyLevels[good])
2063  flatness = (skyLevels[good] - skyMedian) / skyMedian
2064  flatness_rms = numpy.std(flatness)
2065  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2066 
2067  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2068  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2069  (nX, nY, flatness_pp, flatness_rms))
2070 
2071  metadata.set('FLATNESS_PP', float(flatness_pp))
2072  metadata.set('FLATNESS_RMS', float(flatness_rms))
2073  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2074  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2075  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2076 
2077  def roughZeroPoint(self, exposure):
2078  """Set an approximate magnitude zero point for the exposure.
2079 
2080  Parameters
2081  ----------
2082  exposure : `lsst.afw.image.Exposure`
2083  Exposure to process.
2084  """
2085  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2086  if filterName in self.config.fluxMag0T1:
2087  fluxMag0 = self.config.fluxMag0T1[filterName]
2088  else:
2089  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2090  fluxMag0 = self.config.defaultFluxMag0T1
2091 
2092  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2093  if not expTime > 0: # handle NaN as well as <= 0
2094  self.log.warn("Non-positive exposure time; skipping rough zero point")
2095  return
2096 
2097  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2098  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2099 
2100  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2101  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2102 
2103  Parameters
2104  ----------
2105  ccdExposure : `lsst.afw.image.Exposure`
2106  Exposure to process.
2107  fpPolygon : `lsst.afw.geom.Polygon`
2108  Polygon in focal plane coordinates.
2109  """
2110  # Get ccd corners in focal plane coordinates
2111  ccd = ccdExposure.getDetector()
2112  fpCorners = ccd.getCorners(FOCAL_PLANE)
2113  ccdPolygon = Polygon(fpCorners)
2114 
2115  # Get intersection of ccd corners with fpPolygon
2116  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2117 
2118  # Transform back to pixel positions and build new polygon
2119  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2120  validPolygon = Polygon(ccdPoints)
2121  ccdExposure.getInfo().setValidPolygon(validPolygon)
2122 
2123  @contextmanager
2124  def flatContext(self, exp, flat, dark=None):
2125  """Context manager that applies and removes flats and darks,
2126  if the task is configured to apply them.
2127 
2128  Parameters
2129  ----------
2130  exp : `lsst.afw.image.Exposure`
2131  Exposure to process.
2132  flat : `lsst.afw.image.Exposure`
2133  Flat exposure the same size as ``exp``.
2134  dark : `lsst.afw.image.Exposure`, optional
2135  Dark exposure the same size as ``exp``.
2136 
2137  Yields
2138  ------
2139  exp : `lsst.afw.image.Exposure`
2140  The flat and dark corrected exposure.
2141  """
2142  if self.config.doDark and dark is not None:
2143  self.darkCorrection(exp, dark)
2144  if self.config.doFlat:
2145  self.flatCorrection(exp, flat)
2146  try:
2147  yield exp
2148  finally:
2149  if self.config.doFlat:
2150  self.flatCorrection(exp, flat, invert=True)
2151  if self.config.doDark and dark is not None:
2152  self.darkCorrection(exp, dark, invert=True)
2153 
2154  def debugView(self, exposure, stepname):
2155  """Utility function to examine ISR exposure at different stages.
2156 
2157  Parameters
2158  ----------
2159  exposure : `lsst.afw.image.Exposure`
2160  Exposure to view.
2161  stepname : `str`
2162  State of processing to view.
2163  """
2164  frame = getDebugFrame(self._display, stepname)
2165  if frame:
2166  display = getDisplay(frame)
2167  display.scale('asinh', 'zscale')
2168  display.mtv(exposure)
2169 
2170 
2171 class FakeAmp(object):
2172  """A Detector-like object that supports returning gain and saturation level
2173 
2174  This is used when the input exposure does not have a detector.
2175 
2176  Parameters
2177  ----------
2178  exposure : `lsst.afw.image.Exposure`
2179  Exposure to generate a fake amplifier for.
2180  config : `lsst.ip.isr.isrTaskConfig`
2181  Configuration to apply to the fake amplifier.
2182  """
2183 
2184  def __init__(self, exposure, config):
2185  self._bbox = exposure.getBBox(afwImage.LOCAL)
2187  self._gain = config.gain
2188  self._readNoise = config.readNoise
2189  self._saturation = config.saturation
2190 
2191  def getBBox(self):
2192  return self._bbox
2193 
2194  def getRawBBox(self):
2195  return self._bbox
2196 
2197  def getHasRawInfo(self):
2198  return True # but see getRawHorizontalOverscanBBox()
2199 
2201  return self._RawHorizontalOverscanBBox
2202 
2203  def getGain(self):
2204  return self._gain
2205 
2206  def getReadNoise(self):
2207  return self._readNoise
2208 
2209  def getSaturation(self):
2210  return self._saturation
2211 
2212  def getSuspectLevel(self):
2213  return float("NaN")
2214 
2215 
2216 class RunIsrConfig(pexConfig.Config):
2217  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2218 
2219 
2220 class RunIsrTask(pipeBase.CmdLineTask):
2221  """Task to wrap the default IsrTask to allow it to be retargeted.
2222 
2223  The standard IsrTask can be called directly from a command line
2224  program, but doing so removes the ability of the task to be
2225  retargeted. As most cameras override some set of the IsrTask
2226  methods, this would remove those data-specific methods in the
2227  output post-ISR images. This wrapping class fixes the issue,
2228  allowing identical post-ISR images to be generated by both the
2229  processCcd and isrTask code.
2230  """
2231  ConfigClass = RunIsrConfig
2232  _DefaultName = "runIsr"
2233 
2234  def __init__(self, *args, **kwargs):
2235  super().__init__(*args, **kwargs)
2236  self.makeSubtask("isr")
2237 
2238  def runDataRef(self, dataRef):
2239  """
2240  Parameters
2241  ----------
2242  dataRef : `lsst.daf.persistence.ButlerDataRef`
2243  data reference of the detector data to be processed
2244 
2245  Returns
2246  -------
2247  result : `pipeBase.Struct`
2248  Result struct with component:
2249 
2250  - exposure : `lsst.afw.image.Exposure`
2251  Post-ISR processed exposure.
2252  """
2253  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:710
def runDataRef(self, sensorRef)
Definition: isrTask.py:1338
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2013
def debugView(self, exposure, stepname)
Definition: isrTask.py:2154
def __init__(self, kwargs)
Definition: isrTask.py:700
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1431
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:817
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:777
def runDataRef(self, dataRef)
Definition: isrTask.py:2238
def __init__(self, args, kwargs)
Definition: isrTask.py:2234
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1976
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:760
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1877
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2077
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2200
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:747
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1579
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1475
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1828
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:814
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1390
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1766
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1809
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2100
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1511
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:770
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:955
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2124
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1717
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1902
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1937
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1853
def __init__(self, exposure, config)
Definition: isrTask.py:2184