lsst.ip.isr  18.1.0-14-gd04256d+24
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.PrerequisiteInput(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ImageF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ImageF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="MaskedImageF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  bfKernel = cT.PrerequisiteInput(
94  name='bfKernel',
95  doc="Input brighter-fatter kernel.",
96  storageClass="NumpyArray",
97  dimensions=["instrument", "calibration_label"],
98  )
99  defects = cT.PrerequisiteInput(
100  name='defects',
101  doc="Input defect tables.",
102  storageClass="DefectsList",
103  dimensions=["instrument", "calibration_label", "detector"],
104  )
105  opticsTransmission = cT.PrerequisiteInput(
106  name="transmission_optics",
107  storageClass="TransmissionCurve",
108  doc="Transmission curve due to the optics.",
109  dimensions=["instrument", "calibration_label"],
110  )
111  filterTransmission = cT.PrerequisiteInput(
112  name="transmission_filter",
113  storageClass="TransmissionCurve",
114  doc="Transmission curve due to the filter.",
115  dimensions=["instrument", "physical_filter", "calibration_label"],
116  )
117  sensorTransmission = cT.PrerequisiteInput(
118  name="transmission_sensor",
119  storageClass="TransmissionCurve",
120  doc="Transmission curve due to the sensor.",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  atmosphereTransmission = cT.PrerequisiteInput(
124  name="transmission_atmosphere",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the atmosphere.",
127  dimensions=["instrument"],
128  )
129  illumMaskedImage = cT.PrerequisiteInput(
130  name="illum",
131  doc="Input illumination correction.",
132  storageClass="MaskedImageF",
133  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
134  )
135 
136  outputExposure = cT.Output(
137  name='postISRCCD',
138  doc="Output ISR processed exposure.",
139  storageClass="ExposureF",
140  dimensions=["instrument", "visit", "detector"],
141  )
142  preInterpExposure = cT.Output(
143  name='preInterpISRCCD',
144  doc="Output ISR processed exposure, with pixels left uninterpolated.",
145  storageClass="ExposureF",
146  dimensions=["instrument", "visit", "detector"],
147  )
148  outputOssThumbnail = cT.Output(
149  name="OssThumb",
150  doc="Output Overscan-subtracted thumbnail image.",
151  storageClass="Thumbnail",
152  dimensions=["instrument", "visit", "detector"],
153  )
154  outputFlattenedThumbnail = cT.Output(
155  name="FlattenedThumb",
156  doc="Output flat-corrected thumbnail image.",
157  storageClass="Thumbnail",
158  dimensions=["instrument", "visit", "detector"],
159  )
160 
161  def __init__(self, *, config=None):
162  super().__init__(config=config)
163 
164  if config.doBias is not True:
165  self.prerequisiteInputs.discard("bias")
166  if config.doLinearize is not True:
167  self.prerequisiteInputs.discard("linearizer")
168  if config.doCrosstalk is not True:
169  self.prerequisiteInputs.discard("crosstalkSources")
170  if config.doBrighterFatter is not True:
171  self.prerequisiteInputs.discard("bfKernel")
172  if config.doDefect is not True:
173  self.prerequisiteInputs.discard("defects")
174  if config.doDark is not True:
175  self.prerequisiteInputs.discard("dark")
176  if config.doFlat is not True:
177  self.prerequisiteInputs.discard("flat")
178  if config.doAttachTransmissionCurve is not True:
179  self.prerequisiteInputs.discard("opticsTransmission")
180  self.prerequisiteInputs.discard("filterTransmission")
181  self.prerequisiteInputs.discard("sensorTransmission")
182  self.prerequisiteInputs.discard("atmosphereTransmission")
183  if config.doUseOpticsTransmission is not True:
184  self.prerequisiteInputs.discard("opticsTransmission")
185  if config.doUseFilterTransmission is not True:
186  self.prerequisiteInputs.discard("filterTransmission")
187  if config.doUseSensorTransmission is not True:
188  self.prerequisiteInputs.discard("sensorTransmission")
189  if config.doUseAtmosphereTransmission is not True:
190  self.prerequisiteInputs.discard("atmosphereTransmission")
191  if config.doIlluminationCorrection is not True:
192  self.prerequisiteInputs.discard("illumMaskedImage")
193 
194  if config.doWrite is not True:
195  self.outputs.discard("outputExposure")
196  self.outputs.discard("preInterpExposure")
197  self.outputs.discard("outputFlattenedThumbnail")
198  self.outputs.discard("outputOssThumbnail")
199  if config.doSaveInterpPixels is not True:
200  self.outputs.discard("preInterpExposure")
201  if config.qa.doThumbnailOss is not True:
202  self.outputs.discard("outputOssThumbnail")
203  if config.qa.doThumbnailFlattened is not True:
204  self.outputs.discard("outputFlattenedThumbnail")
205 
206 
207 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
208  pipelineConnections=IsrTaskConnections):
209  """Configuration parameters for IsrTask.
210 
211  Items are grouped in the order in which they are executed by the task.
212  """
213  datasetType = pexConfig.Field(
214  dtype=str,
215  doc="Dataset type for input data; users will typically leave this alone, "
216  "but camera-specific ISR tasks will override it",
217  default="raw",
218  )
219 
220  fallbackFilterName = pexConfig.Field(
221  dtype=str,
222  doc="Fallback default filter name for calibrations.",
223  optional=True
224  )
225  expectWcs = pexConfig.Field(
226  dtype=bool,
227  default=True,
228  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
229  )
230  fwhm = pexConfig.Field(
231  dtype=float,
232  doc="FWHM of PSF in arcseconds.",
233  default=1.0,
234  )
235  qa = pexConfig.ConfigField(
236  dtype=isrQa.IsrQaConfig,
237  doc="QA related configuration options.",
238  )
239 
240  # Image conversion configuration
241  doConvertIntToFloat = pexConfig.Field(
242  dtype=bool,
243  doc="Convert integer raw images to floating point values?",
244  default=True,
245  )
246 
247  # Saturated pixel handling.
248  doSaturation = pexConfig.Field(
249  dtype=bool,
250  doc="Mask saturated pixels? NB: this is totally independent of the"
251  " interpolation option - this is ONLY setting the bits in the mask."
252  " To have them interpolated make sure doSaturationInterpolation=True",
253  default=True,
254  )
255  saturatedMaskName = pexConfig.Field(
256  dtype=str,
257  doc="Name of mask plane to use in saturation detection and interpolation",
258  default="SAT",
259  )
260  saturation = pexConfig.Field(
261  dtype=float,
262  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
263  default=float("NaN"),
264  )
265  growSaturationFootprintSize = pexConfig.Field(
266  dtype=int,
267  doc="Number of pixels by which to grow the saturation footprints",
268  default=1,
269  )
270 
271  # Suspect pixel handling.
272  doSuspect = pexConfig.Field(
273  dtype=bool,
274  doc="Mask suspect pixels?",
275  default=True,
276  )
277  suspectMaskName = pexConfig.Field(
278  dtype=str,
279  doc="Name of mask plane to use for suspect pixels",
280  default="SUSPECT",
281  )
282  numEdgeSuspect = pexConfig.Field(
283  dtype=int,
284  doc="Number of edge pixels to be flagged as untrustworthy.",
285  default=0,
286  )
287 
288  # Initial masking options.
289  doSetBadRegions = pexConfig.Field(
290  dtype=bool,
291  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
292  default=True,
293  )
294  badStatistic = pexConfig.ChoiceField(
295  dtype=str,
296  doc="How to estimate the average value for BAD regions.",
297  default='MEANCLIP',
298  allowed={
299  "MEANCLIP": "Correct using the (clipped) mean of good data",
300  "MEDIAN": "Correct using the median of the good data",
301  },
302  )
303 
304  # Overscan subtraction configuration.
305  doOverscan = pexConfig.Field(
306  dtype=bool,
307  doc="Do overscan subtraction?",
308  default=True,
309  )
310  overscanFitType = pexConfig.ChoiceField(
311  dtype=str,
312  doc="The method for fitting the overscan bias level.",
313  default='MEDIAN',
314  allowed={
315  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
316  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
317  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
318  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
319  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
320  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
321  "MEAN": "Correct using the mean of the overscan region",
322  "MEANCLIP": "Correct using a clipped mean of the overscan region",
323  "MEDIAN": "Correct using the median of the overscan region",
324  },
325  )
326  overscanOrder = pexConfig.Field(
327  dtype=int,
328  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
329  "or number of spline knots if overscan fit type is a spline."),
330  default=1,
331  )
332  overscanNumSigmaClip = pexConfig.Field(
333  dtype=float,
334  doc="Rejection threshold (sigma) for collapsing overscan before fit",
335  default=3.0,
336  )
337  overscanIsInt = pexConfig.Field(
338  dtype=bool,
339  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
340  default=True,
341  )
342  overscanNumLeadingColumnsToSkip = pexConfig.Field(
343  dtype=int,
344  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
345  default=0,
346  )
347  overscanNumTrailingColumnsToSkip = pexConfig.Field(
348  dtype=int,
349  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
350  default=0,
351  )
352  overscanMaxDev = pexConfig.Field(
353  dtype=float,
354  doc="Maximum deviation from the median for overscan",
355  default=1000.0, check=lambda x: x > 0
356  )
357  overscanBiasJump = pexConfig.Field(
358  dtype=bool,
359  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
360  default=False,
361  )
362  overscanBiasJumpKeyword = pexConfig.Field(
363  dtype=str,
364  doc="Header keyword containing information about devices.",
365  default="NO_SUCH_KEY",
366  )
367  overscanBiasJumpDevices = pexConfig.ListField(
368  dtype=str,
369  doc="List of devices that need piecewise overscan correction.",
370  default=(),
371  )
372  overscanBiasJumpLocation = pexConfig.Field(
373  dtype=int,
374  doc="Location of bias jump along y-axis.",
375  default=0,
376  )
377 
378  # Amplifier to CCD assembly configuration
379  doAssembleCcd = pexConfig.Field(
380  dtype=bool,
381  default=True,
382  doc="Assemble amp-level exposures into a ccd-level exposure?"
383  )
384  assembleCcd = pexConfig.ConfigurableField(
385  target=AssembleCcdTask,
386  doc="CCD assembly task",
387  )
388 
389  # General calibration configuration.
390  doAssembleIsrExposures = pexConfig.Field(
391  dtype=bool,
392  default=False,
393  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
394  )
395  doTrimToMatchCalib = pexConfig.Field(
396  dtype=bool,
397  default=False,
398  doc="Trim raw data to match calibration bounding boxes?"
399  )
400 
401  # Bias subtraction.
402  doBias = pexConfig.Field(
403  dtype=bool,
404  doc="Apply bias frame correction?",
405  default=True,
406  )
407  biasDataProductName = pexConfig.Field(
408  dtype=str,
409  doc="Name of the bias data product",
410  default="bias",
411  )
412 
413  # Variance construction
414  doVariance = pexConfig.Field(
415  dtype=bool,
416  doc="Calculate variance?",
417  default=True
418  )
419  gain = pexConfig.Field(
420  dtype=float,
421  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
422  default=float("NaN"),
423  )
424  readNoise = pexConfig.Field(
425  dtype=float,
426  doc="The read noise to use if no Detector is present in the Exposure",
427  default=0.0,
428  )
429  doEmpiricalReadNoise = pexConfig.Field(
430  dtype=bool,
431  default=False,
432  doc="Calculate empirical read noise instead of value from AmpInfo data?"
433  )
434 
435  # Linearization.
436  doLinearize = pexConfig.Field(
437  dtype=bool,
438  doc="Correct for nonlinearity of the detector's response?",
439  default=True,
440  )
441 
442  # Crosstalk.
443  doCrosstalk = pexConfig.Field(
444  dtype=bool,
445  doc="Apply intra-CCD crosstalk correction?",
446  default=False,
447  )
448  doCrosstalkBeforeAssemble = pexConfig.Field(
449  dtype=bool,
450  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
451  default=False,
452  )
453  crosstalk = pexConfig.ConfigurableField(
454  target=CrosstalkTask,
455  doc="Intra-CCD crosstalk correction",
456  )
457 
458  # Masking options.
459  doDefect = pexConfig.Field(
460  dtype=bool,
461  doc="Apply correction for CCD defects, e.g. hot pixels?",
462  default=True,
463  )
464  doNanMasking = pexConfig.Field(
465  dtype=bool,
466  doc="Mask NAN pixels?",
467  default=True,
468  )
469  doWidenSaturationTrails = pexConfig.Field(
470  dtype=bool,
471  doc="Widen bleed trails based on their width?",
472  default=True
473  )
474 
475  # Brighter-Fatter correction.
476  doBrighterFatter = pexConfig.Field(
477  dtype=bool,
478  default=False,
479  doc="Apply the brighter fatter correction"
480  )
481  brighterFatterLevel = pexConfig.ChoiceField(
482  dtype=str,
483  default="DETECTOR",
484  doc="The level at which to correct for brighter-fatter.",
485  allowed={
486  "AMP": "Every amplifier treated separately.",
487  "DETECTOR": "One kernel per detector",
488  }
489  )
490  brighterFatterKernelFile = pexConfig.Field(
491  dtype=str,
492  default='',
493  doc="Kernel file used for the brighter fatter correction"
494  )
495  brighterFatterMaxIter = pexConfig.Field(
496  dtype=int,
497  default=10,
498  doc="Maximum number of iterations for the brighter fatter correction"
499  )
500  brighterFatterThreshold = pexConfig.Field(
501  dtype=float,
502  default=1000,
503  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
504  " absolute value of the difference between the current corrected image and the one"
505  " from the previous iteration summed over all the pixels."
506  )
507  brighterFatterApplyGain = pexConfig.Field(
508  dtype=bool,
509  default=True,
510  doc="Should the gain be applied when applying the brighter fatter correction?"
511  )
512 
513  # Dark subtraction.
514  doDark = pexConfig.Field(
515  dtype=bool,
516  doc="Apply dark frame correction?",
517  default=True,
518  )
519  darkDataProductName = pexConfig.Field(
520  dtype=str,
521  doc="Name of the dark data product",
522  default="dark",
523  )
524 
525  # Camera-specific stray light removal.
526  doStrayLight = pexConfig.Field(
527  dtype=bool,
528  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
529  default=False,
530  )
531  strayLight = pexConfig.ConfigurableField(
532  target=StrayLightTask,
533  doc="y-band stray light correction"
534  )
535 
536  # Flat correction.
537  doFlat = pexConfig.Field(
538  dtype=bool,
539  doc="Apply flat field correction?",
540  default=True,
541  )
542  flatDataProductName = pexConfig.Field(
543  dtype=str,
544  doc="Name of the flat data product",
545  default="flat",
546  )
547  flatScalingType = pexConfig.ChoiceField(
548  dtype=str,
549  doc="The method for scaling the flat on the fly.",
550  default='USER',
551  allowed={
552  "USER": "Scale by flatUserScale",
553  "MEAN": "Scale by the inverse of the mean",
554  "MEDIAN": "Scale by the inverse of the median",
555  },
556  )
557  flatUserScale = pexConfig.Field(
558  dtype=float,
559  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
560  default=1.0,
561  )
562  doTweakFlat = pexConfig.Field(
563  dtype=bool,
564  doc="Tweak flats to match observed amplifier ratios?",
565  default=False
566  )
567 
568  # Amplifier normalization based on gains instead of using flats configuration.
569  doApplyGains = pexConfig.Field(
570  dtype=bool,
571  doc="Correct the amplifiers for their gains instead of applying flat correction",
572  default=False,
573  )
574  normalizeGains = pexConfig.Field(
575  dtype=bool,
576  doc="Normalize all the amplifiers in each CCD to have the same median value.",
577  default=False,
578  )
579 
580  # Fringe correction.
581  doFringe = pexConfig.Field(
582  dtype=bool,
583  doc="Apply fringe correction?",
584  default=True,
585  )
586  fringe = pexConfig.ConfigurableField(
587  target=FringeTask,
588  doc="Fringe subtraction task",
589  )
590  fringeAfterFlat = pexConfig.Field(
591  dtype=bool,
592  doc="Do fringe subtraction after flat-fielding?",
593  default=True,
594  )
595 
596  # Distortion model application.
597  doAddDistortionModel = pexConfig.Field(
598  dtype=bool,
599  doc="Apply a distortion model based on camera geometry to the WCS?",
600  default=True,
601  deprecated=("Camera geometry is incorporated when reading the raw files."
602  " This option no longer is used, and will be removed after v19.")
603  )
604 
605  # Initial CCD-level background statistics options.
606  doMeasureBackground = pexConfig.Field(
607  dtype=bool,
608  doc="Measure the background level on the reduced image?",
609  default=False,
610  )
611 
612  # Camera-specific masking configuration.
613  doCameraSpecificMasking = pexConfig.Field(
614  dtype=bool,
615  doc="Mask camera-specific bad regions?",
616  default=False,
617  )
618  masking = pexConfig.ConfigurableField(
619  target=MaskingTask,
620  doc="Masking task."
621  )
622 
623  # Interpolation options.
624 
625  doInterpolate = pexConfig.Field(
626  dtype=bool,
627  doc="Interpolate masked pixels?",
628  default=True,
629  )
630  doSaturationInterpolation = pexConfig.Field(
631  dtype=bool,
632  doc="Perform interpolation over pixels masked as saturated?"
633  " NB: This is independent of doSaturation; if that is False this plane"
634  " will likely be blank, resulting in a no-op here.",
635  default=True,
636  )
637  doNanInterpolation = pexConfig.Field(
638  dtype=bool,
639  doc="Perform interpolation over pixels masked as NaN?"
640  " NB: This is independent of doNanMasking; if that is False this plane"
641  " will likely be blank, resulting in a no-op here.",
642  default=True,
643  )
644  doNanInterpAfterFlat = pexConfig.Field(
645  dtype=bool,
646  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
647  "also have to interpolate them before flat-fielding."),
648  default=False,
649  )
650  maskListToInterpolate = pexConfig.ListField(
651  dtype=str,
652  doc="List of mask planes that should be interpolated.",
653  default=['SAT', 'BAD', 'UNMASKEDNAN'],
654  )
655  doSaveInterpPixels = pexConfig.Field(
656  dtype=bool,
657  doc="Save a copy of the pre-interpolated pixel values?",
658  default=False,
659  )
660 
661  # Default photometric calibration options.
662  fluxMag0T1 = pexConfig.DictField(
663  keytype=str,
664  itemtype=float,
665  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
666  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
667  ))
668  )
669  defaultFluxMag0T1 = pexConfig.Field(
670  dtype=float,
671  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
672  default=pow(10.0, 0.4*28.0)
673  )
674 
675  # Vignette correction configuration.
676  doVignette = pexConfig.Field(
677  dtype=bool,
678  doc="Apply vignetting parameters?",
679  default=False,
680  )
681  vignette = pexConfig.ConfigurableField(
682  target=VignetteTask,
683  doc="Vignetting task.",
684  )
685 
686  # Transmission curve configuration.
687  doAttachTransmissionCurve = pexConfig.Field(
688  dtype=bool,
689  default=False,
690  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
691  )
692  doUseOpticsTransmission = pexConfig.Field(
693  dtype=bool,
694  default=True,
695  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
696  )
697  doUseFilterTransmission = pexConfig.Field(
698  dtype=bool,
699  default=True,
700  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
701  )
702  doUseSensorTransmission = pexConfig.Field(
703  dtype=bool,
704  default=True,
705  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
706  )
707  doUseAtmosphereTransmission = pexConfig.Field(
708  dtype=bool,
709  default=True,
710  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
711  )
712 
713  # Illumination correction.
714  doIlluminationCorrection = pexConfig.Field(
715  dtype=bool,
716  default=False,
717  doc="Perform illumination correction?"
718  )
719  illuminationCorrectionDataProductName = pexConfig.Field(
720  dtype=str,
721  doc="Name of the illumination correction data product.",
722  default="illumcor",
723  )
724  illumScale = pexConfig.Field(
725  dtype=float,
726  doc="Scale factor for the illumination correction.",
727  default=1.0,
728  )
729  illumFilters = pexConfig.ListField(
730  dtype=str,
731  default=[],
732  doc="Only perform illumination correction for these filters."
733  )
734 
735  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
736  doWrite = pexConfig.Field(
737  dtype=bool,
738  doc="Persist postISRCCD?",
739  default=True,
740  )
741 
742  def validate(self):
743  super().validate()
744  if self.doFlat and self.doApplyGains:
745  raise ValueError("You may not specify both doFlat and doApplyGains")
746  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
747  self.config.maskListToInterpolate.append("SAT")
748  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
749  self.config.maskListToInterpolate.append("UNMASKEDNAN")
750 
751 
752 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
753  """Apply common instrument signature correction algorithms to a raw frame.
754 
755  The process for correcting imaging data is very similar from
756  camera to camera. This task provides a vanilla implementation of
757  doing these corrections, including the ability to turn certain
758  corrections off if they are not needed. The inputs to the primary
759  method, `run()`, are a raw exposure to be corrected and the
760  calibration data products. The raw input is a single chip sized
761  mosaic of all amps including overscans and other non-science
762  pixels. The method `runDataRef()` identifies and defines the
763  calibration data products, and is intended for use by a
764  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
765  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
766  subclassed for different camera, although the most camera specific
767  methods have been split into subtasks that can be redirected
768  appropriately.
769 
770  The __init__ method sets up the subtasks for ISR processing, using
771  the defaults from `lsst.ip.isr`.
772 
773  Parameters
774  ----------
775  args : `list`
776  Positional arguments passed to the Task constructor. None used at this time.
777  kwargs : `dict`, optional
778  Keyword arguments passed on to the Task constructor. None used at this time.
779  """
780  ConfigClass = IsrTaskConfig
781  _DefaultName = "isr"
782 
783  def __init__(self, **kwargs):
784  super().__init__(**kwargs)
785  self.makeSubtask("assembleCcd")
786  self.makeSubtask("crosstalk")
787  self.makeSubtask("strayLight")
788  self.makeSubtask("fringe")
789  self.makeSubtask("masking")
790  self.makeSubtask("vignette")
791 
792  def runQuantum(self, butlerQC, inputRefs, outputRefs):
793  inputs = butlerQC.get(inputRefs)
794 
795  try:
796  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
797  except Exception as e:
798  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
799  (inputRefs, e))
800 
801  inputs['isGen3'] = True
802 
803  if self.config.doLinearize is True:
804  if 'linearizer' not in inputs.keys():
805  detector = inputs['camera'][inputs['detectorNum']]
806  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
807  inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
808 
809  if inputs['defects'] is not None:
810  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
811  # masking expects a list of defects defined by their bounding box
812  if not isinstance(inputs["defects"], Defects):
813  inputs["defects"] = Defects.fromTable(inputs["defects"])
814 
815  # Broken: DM-17169
816  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
817  # fixed for non-HSC cameras in the future.
818  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
819  # if self.config.doCrosstalk else None)
820 
821  # Broken: DM-17152
822  # Fringes are not tested to be handled correctly by Gen3 butler.
823  # inputs['fringes'] = (self.fringe.readFringes(inputsIds['ccdExposure'],
824  # assembler=self.assembleCcd
825  # if self.config.doAssembleIsrExposures else None)
826  # if self.config.doFringe and
827  # self.fringe.checkFilter(inputs['ccdExposure'])
828  # else pipeBase.Struct(fringes=None))
829 
830  outputs = self.run(**inputs)
831  butlerQC.put(outputs, outputRefs)
832 
833  def readIsrData(self, dataRef, rawExposure):
834  """!Retrieve necessary frames for instrument signature removal.
835 
836  Pre-fetching all required ISR data products limits the IO
837  required by the ISR. Any conflict between the calibration data
838  available and that needed for ISR is also detected prior to
839  doing processing, allowing it to fail quickly.
840 
841  Parameters
842  ----------
843  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
844  Butler reference of the detector data to be processed
845  rawExposure : `afw.image.Exposure`
846  The raw exposure that will later be corrected with the
847  retrieved calibration data; should not be modified in this
848  method.
849 
850  Returns
851  -------
852  result : `lsst.pipe.base.Struct`
853  Result struct with components (which may be `None`):
854  - ``bias``: bias calibration frame (`afw.image.Exposure`)
855  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
856  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
857  - ``dark``: dark calibration frame (`afw.image.Exposure`)
858  - ``flat``: flat calibration frame (`afw.image.Exposure`)
859  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
860  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
861  - ``fringes``: `lsst.pipe.base.Struct` with components:
862  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
863  - ``seed``: random seed derived from the ccdExposureId for random
864  number generator (`uint32`).
865  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
866  A ``TransmissionCurve`` that represents the throughput of the optics,
867  to be evaluated in focal-plane coordinates.
868  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
869  A ``TransmissionCurve`` that represents the throughput of the filter
870  itself, to be evaluated in focal-plane coordinates.
871  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
872  A ``TransmissionCurve`` that represents the throughput of the sensor
873  itself, to be evaluated in post-assembly trimmed detector coordinates.
874  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
875  A ``TransmissionCurve`` that represents the throughput of the
876  atmosphere, assumed to be spatially constant.
877  - ``strayLightData`` : `object`
878  An opaque object containing calibration information for
879  stray-light correction. If `None`, no correction will be
880  performed.
881  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
882 
883  Raises
884  ------
885  NotImplementedError :
886  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
887  """
888  ccd = rawExposure.getDetector()
889  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
890  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
891  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
892  if self.config.doBias else None)
893  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
894  linearizer = (dataRef.get("linearizer", immediate=True)
895  if self.doLinearize(ccd) else None)
896  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
897  if self.config.doCrosstalk else None)
898  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
899  if self.config.doDark else None)
900  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
901  if self.config.doFlat else None)
902 
903  brighterFatterKernel = None
904  if self.config.doBrighterFatter is True:
905 
906  # Use the new-style cp_pipe version of the kernel is it exists.
907  try:
908  brighterFatterKernel = dataRef.get("brighterFatterKernel")
909  except NoResults:
910  # Fall back to the old-style numpy-ndarray style kernel if necessary.
911  try:
912  brighterFatterKernel = dataRef.get("bfKernel")
913  except NoResults:
914  brighterFatterKernel = None
915  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
916  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
917  # this detector, or raise an error.
918  if self.config.brighterFatterLevel == 'DETECTOR':
919  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
920  else:
921  # TODO DM-15631 for implementing this
922  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
923 
924  defectList = (dataRef.get("defects")
925  if self.config.doDefect else None)
926  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
927  if self.config.doAssembleIsrExposures else None)
928  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
929  else pipeBase.Struct(fringes=None))
930 
931  if self.config.doAttachTransmissionCurve:
932  opticsTransmission = (dataRef.get("transmission_optics")
933  if self.config.doUseOpticsTransmission else None)
934  filterTransmission = (dataRef.get("transmission_filter")
935  if self.config.doUseFilterTransmission else None)
936  sensorTransmission = (dataRef.get("transmission_sensor")
937  if self.config.doUseSensorTransmission else None)
938  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
939  if self.config.doUseAtmosphereTransmission else None)
940  else:
941  opticsTransmission = None
942  filterTransmission = None
943  sensorTransmission = None
944  atmosphereTransmission = None
945 
946  if self.config.doStrayLight:
947  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
948  else:
949  strayLightData = None
950 
951  illumMaskedImage = (self.getIsrExposure(dataRef,
952  self.config.illuminationCorrectionDataProductName).getMaskedImage()
953  if (self.config.doIlluminationCorrection and
954  filterName in self.config.illumFilters)
955  else None)
956 
957  # Struct should include only kwargs to run()
958  return pipeBase.Struct(bias=biasExposure,
959  linearizer=linearizer,
960  crosstalkSources=crosstalkSources,
961  dark=darkExposure,
962  flat=flatExposure,
963  bfKernel=brighterFatterKernel,
964  defects=defectList,
965  fringes=fringeStruct,
966  opticsTransmission=opticsTransmission,
967  filterTransmission=filterTransmission,
968  sensorTransmission=sensorTransmission,
969  atmosphereTransmission=atmosphereTransmission,
970  strayLightData=strayLightData,
971  illumMaskedImage=illumMaskedImage
972  )
973 
974  @pipeBase.timeMethod
975  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
976  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
977  opticsTransmission=None, filterTransmission=None,
978  sensorTransmission=None, atmosphereTransmission=None,
979  detectorNum=None, strayLightData=None, illumMaskedImage=None,
980  isGen3=False,
981  ):
982  """!Perform instrument signature removal on an exposure.
983 
984  Steps included in the ISR processing, in order performed, are:
985  - saturation and suspect pixel masking
986  - overscan subtraction
987  - CCD assembly of individual amplifiers
988  - bias subtraction
989  - variance image construction
990  - linearization of non-linear response
991  - crosstalk masking
992  - brighter-fatter correction
993  - dark subtraction
994  - fringe correction
995  - stray light subtraction
996  - flat correction
997  - masking of known defects and camera specific features
998  - vignette calculation
999  - appending transmission curve and distortion model
1000 
1001  Parameters
1002  ----------
1003  ccdExposure : `lsst.afw.image.Exposure`
1004  The raw exposure that is to be run through ISR. The
1005  exposure is modified by this method.
1006  camera : `lsst.afw.cameraGeom.Camera`, optional
1007  The camera geometry for this exposure. Used to select the
1008  distortion model appropriate for this data.
1009  bias : `lsst.afw.image.Exposure`, optional
1010  Bias calibration frame.
1011  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1012  Functor for linearization.
1013  crosstalkSources : `list`, optional
1014  List of possible crosstalk sources.
1015  dark : `lsst.afw.image.Exposure`, optional
1016  Dark calibration frame.
1017  flat : `lsst.afw.image.Exposure`, optional
1018  Flat calibration frame.
1019  bfKernel : `numpy.ndarray`, optional
1020  Brighter-fatter kernel.
1021  defects : `lsst.meas.algorithms.Defects`, optional
1022  List of defects.
1023  fringes : `lsst.pipe.base.Struct`, optional
1024  Struct containing the fringe correction data, with
1025  elements:
1026  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1027  - ``seed``: random seed derived from the ccdExposureId for random
1028  number generator (`uint32`)
1029  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1030  A ``TransmissionCurve`` that represents the throughput of the optics,
1031  to be evaluated in focal-plane coordinates.
1032  filterTransmission : `lsst.afw.image.TransmissionCurve`
1033  A ``TransmissionCurve`` that represents the throughput of the filter
1034  itself, to be evaluated in focal-plane coordinates.
1035  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1036  A ``TransmissionCurve`` that represents the throughput of the sensor
1037  itself, to be evaluated in post-assembly trimmed detector coordinates.
1038  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1039  A ``TransmissionCurve`` that represents the throughput of the
1040  atmosphere, assumed to be spatially constant.
1041  detectorNum : `int`, optional
1042  The integer number for the detector to process.
1043  isGen3 : bool, optional
1044  Flag this call to run() as using the Gen3 butler environment.
1045  strayLightData : `object`, optional
1046  Opaque object containing calibration information for stray-light
1047  correction. If `None`, no correction will be performed.
1048  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1049  Illumination correction image.
1050 
1051  Returns
1052  -------
1053  result : `lsst.pipe.base.Struct`
1054  Result struct with component:
1055  - ``exposure`` : `afw.image.Exposure`
1056  The fully ISR corrected exposure.
1057  - ``outputExposure`` : `afw.image.Exposure`
1058  An alias for `exposure`
1059  - ``ossThumb`` : `numpy.ndarray`
1060  Thumbnail image of the exposure after overscan subtraction.
1061  - ``flattenedThumb`` : `numpy.ndarray`
1062  Thumbnail image of the exposure after flat-field correction.
1063 
1064  Raises
1065  ------
1066  RuntimeError
1067  Raised if a configuration option is set to True, but the
1068  required calibration data has not been specified.
1069 
1070  Notes
1071  -----
1072  The current processed exposure can be viewed by setting the
1073  appropriate lsstDebug entries in the `debug.display`
1074  dictionary. The names of these entries correspond to some of
1075  the IsrTaskConfig Boolean options, with the value denoting the
1076  frame to use. The exposure is shown inside the matching
1077  option check and after the processing of that step has
1078  finished. The steps with debug points are:
1079 
1080  doAssembleCcd
1081  doBias
1082  doCrosstalk
1083  doBrighterFatter
1084  doDark
1085  doFringe
1086  doStrayLight
1087  doFlat
1088 
1089  In addition, setting the "postISRCCD" entry displays the
1090  exposure after all ISR processing has finished.
1091 
1092  """
1093 
1094  if isGen3 is True:
1095  # Gen3 currently cannot automatically do configuration overrides.
1096  # DM-15257 looks to discuss this issue.
1097 
1098  self.config.doFringe = False
1099 
1100  # Configure input exposures;
1101  if detectorNum is None:
1102  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1103 
1104  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1105  bias = self.ensureExposure(bias, camera, detectorNum)
1106  dark = self.ensureExposure(dark, camera, detectorNum)
1107  flat = self.ensureExposure(flat, camera, detectorNum)
1108  else:
1109  if isinstance(ccdExposure, ButlerDataRef):
1110  return self.runDataRef(ccdExposure)
1111 
1112  ccd = ccdExposure.getDetector()
1113  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1114 
1115  if not ccd:
1116  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1117  ccd = [FakeAmp(ccdExposure, self.config)]
1118 
1119  # Validate Input
1120  if self.config.doBias and bias is None:
1121  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1122  if self.doLinearize(ccd) and linearizer is None:
1123  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1124  if self.config.doBrighterFatter and bfKernel is None:
1125  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1126  if self.config.doDark and dark is None:
1127  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1128  if self.config.doFlat and flat is None:
1129  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1130  if self.config.doDefect and defects is None:
1131  raise RuntimeError("Must supply defects if config.doDefect=True.")
1132  if (self.config.doFringe and filterName in self.fringe.config.filters and
1133  fringes.fringes is None):
1134  # The `fringes` object needs to be a pipeBase.Struct, as
1135  # we use it as a `dict` for the parameters of
1136  # `FringeTask.run()`. The `fringes.fringes` `list` may
1137  # not be `None` if `doFringe=True`. Otherwise, raise.
1138  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1139  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1140  illumMaskedImage is None):
1141  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1142 
1143  # Begin ISR processing.
1144  if self.config.doConvertIntToFloat:
1145  self.log.info("Converting exposure to floating point values.")
1146  ccdExposure = self.convertIntToFloat(ccdExposure)
1147 
1148  # Amplifier level processing.
1149  overscans = []
1150  for amp in ccd:
1151  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1152  if ccdExposure.getBBox().contains(amp.getBBox()):
1153  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1154  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1155 
1156  if self.config.doOverscan and not badAmp:
1157  # Overscan correction on amp-by-amp basis.
1158  overscanResults = self.overscanCorrection(ccdExposure, amp)
1159  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1160  if overscanResults is not None and \
1161  self.config.qa is not None and self.config.qa.saveStats is True:
1162  if isinstance(overscanResults.overscanFit, float):
1163  qaMedian = overscanResults.overscanFit
1164  qaStdev = float("NaN")
1165  else:
1166  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1167  afwMath.MEDIAN | afwMath.STDEVCLIP)
1168  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1169  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1170 
1171  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1172  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1173  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1174  amp.getName(), qaMedian, qaStdev)
1175  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1176  else:
1177  if badAmp:
1178  self.log.warn("Amplifier %s is bad.", amp.getName())
1179  overscanResults = None
1180 
1181  overscans.append(overscanResults if overscanResults is not None else None)
1182  else:
1183  self.log.info("Skipped OSCAN for %s.", amp.getName())
1184 
1185  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1186  self.log.info("Applying crosstalk correction.")
1187  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1188  self.debugView(ccdExposure, "doCrosstalk")
1189 
1190  if self.config.doAssembleCcd:
1191  self.log.info("Assembling CCD from amplifiers.")
1192  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1193 
1194  if self.config.expectWcs and not ccdExposure.getWcs():
1195  self.log.warn("No WCS found in input exposure.")
1196  self.debugView(ccdExposure, "doAssembleCcd")
1197 
1198  ossThumb = None
1199  if self.config.qa.doThumbnailOss:
1200  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1201 
1202  if self.config.doBias:
1203  self.log.info("Applying bias correction.")
1204  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1205  trimToFit=self.config.doTrimToMatchCalib)
1206  self.debugView(ccdExposure, "doBias")
1207 
1208  if self.config.doVariance:
1209  for amp, overscanResults in zip(ccd, overscans):
1210  if ccdExposure.getBBox().contains(amp.getBBox()):
1211  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1212  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1213  if overscanResults is not None:
1214  self.updateVariance(ampExposure, amp,
1215  overscanImage=overscanResults.overscanImage)
1216  else:
1217  self.updateVariance(ampExposure, amp,
1218  overscanImage=None)
1219  if self.config.qa is not None and self.config.qa.saveStats is True:
1220  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1221  afwMath.MEDIAN | afwMath.STDEVCLIP)
1222  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1223  qaStats.getValue(afwMath.MEDIAN))
1224  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1225  qaStats.getValue(afwMath.STDEVCLIP))
1226  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1227  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1228  qaStats.getValue(afwMath.STDEVCLIP))
1229 
1230  if self.doLinearize(ccd):
1231  self.log.info("Applying linearizer.")
1232  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1233 
1234  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1235  self.log.info("Applying crosstalk correction.")
1236  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1237  self.debugView(ccdExposure, "doCrosstalk")
1238 
1239  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1240  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1241  if self.config.doDefect:
1242  self.log.info("Masking defects.")
1243  self.maskDefect(ccdExposure, defects)
1244 
1245  if self.config.numEdgeSuspect > 0:
1246  self.log.info("Masking edges as SUSPECT.")
1247  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1248  maskPlane="SUSPECT")
1249 
1250  if self.config.doNanMasking:
1251  self.log.info("Masking NAN value pixels.")
1252  self.maskNan(ccdExposure)
1253 
1254  if self.config.doWidenSaturationTrails:
1255  self.log.info("Widening saturation trails.")
1256  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1257 
1258  if self.config.doCameraSpecificMasking:
1259  self.log.info("Masking regions for camera specific reasons.")
1260  self.masking.run(ccdExposure)
1261 
1262  if self.config.doBrighterFatter:
1263  # We need to apply flats and darks before we can interpolate, and we
1264  # need to interpolate before we do B-F, but we do B-F without the
1265  # flats and darks applied so we can work in units of electrons or holes.
1266  # This context manager applies and then removes the darks and flats.
1267  #
1268  # We also do not want to interpolate values here, so operate on temporary
1269  # images so we can apply only the BF-correction and roll back the
1270  # interpolation.
1271  interpExp = ccdExposure.clone()
1272  with self.flatContext(interpExp, flat, dark):
1273  isrFunctions.interpolateFromMask(
1274  maskedImage=interpExp.getMaskedImage(),
1275  fwhm=self.config.fwhm,
1276  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1277  maskNameList=self.config.maskListToInterpolate
1278  )
1279  bfExp = interpExp.clone()
1280 
1281  self.log.info("Applying brighter fatter correction.")
1282  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1283  self.config.brighterFatterMaxIter,
1284  self.config.brighterFatterThreshold,
1285  self.config.brighterFatterApplyGain
1286  )
1287  if bfResults[1] == self.config.brighterFatterMaxIter:
1288  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1289  bfResults[0])
1290  else:
1291  self.log.info("Finished brighter fatter correction in %d iterations.",
1292  bfResults[1])
1293  image = ccdExposure.getMaskedImage().getImage()
1294  bfCorr = bfExp.getMaskedImage().getImage()
1295  bfCorr -= interpExp.getMaskedImage().getImage()
1296  image += bfCorr
1297 
1298  # Applying the brighter-fatter correction applies a
1299  # convolution to the science image. At the edges this
1300  # convolution may not have sufficient valid pixels to
1301  # produce a valid correction. Mark pixels within the size
1302  # of the brighter-fatter kernel as EDGE to warn of this
1303  # fact.
1304  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1305  maskPlane="EDGE")
1306  self.log.warn("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1307 
1308  self.debugView(ccdExposure, "doBrighterFatter")
1309 
1310  if self.config.doDark:
1311  self.log.info("Applying dark correction.")
1312  self.darkCorrection(ccdExposure, dark)
1313  self.debugView(ccdExposure, "doDark")
1314 
1315  if self.config.doFringe and not self.config.fringeAfterFlat:
1316  self.log.info("Applying fringe correction before flat.")
1317  self.fringe.run(ccdExposure, **fringes.getDict())
1318  self.debugView(ccdExposure, "doFringe")
1319 
1320  if self.config.doStrayLight:
1321  if strayLightData is not None:
1322  self.log.info("Applying stray light correction.")
1323  self.strayLight.run(ccdExposure, strayLightData)
1324  self.debugView(ccdExposure, "doStrayLight")
1325  else:
1326  self.log.debug("Skipping stray light correction: no data found for this image.")
1327 
1328  if self.config.doFlat:
1329  self.log.info("Applying flat correction.")
1330  self.flatCorrection(ccdExposure, flat)
1331  self.debugView(ccdExposure, "doFlat")
1332 
1333  if self.config.doApplyGains:
1334  self.log.info("Applying gain correction instead of flat.")
1335  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1336 
1337  if self.config.doFringe and self.config.fringeAfterFlat:
1338  self.log.info("Applying fringe correction after flat.")
1339  self.fringe.run(ccdExposure, **fringes.getDict())
1340 
1341  if self.config.doVignette:
1342  self.log.info("Constructing Vignette polygon.")
1343  self.vignettePolygon = self.vignette.run(ccdExposure)
1344 
1345  if self.config.vignette.doWriteVignettePolygon:
1346  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1347 
1348  if self.config.doAttachTransmissionCurve:
1349  self.log.info("Adding transmission curves.")
1350  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1351  filterTransmission=filterTransmission,
1352  sensorTransmission=sensorTransmission,
1353  atmosphereTransmission=atmosphereTransmission)
1354 
1355  flattenedThumb = None
1356  if self.config.qa.doThumbnailFlattened:
1357  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1358 
1359  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1360  self.log.info("Performing illumination correction.")
1361  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1362  illumMaskedImage, illumScale=self.config.illumScale,
1363  trimToFit=self.config.doTrimToMatchCalib)
1364 
1365  preInterpExp = None
1366  if self.config.doSaveInterpPixels:
1367  preInterpExp = ccdExposure.clone()
1368 
1369  # Reset and interpolate bad pixels.
1370  #
1371  # Large contiguous bad regions (which should have the BAD mask
1372  # bit set) should have their values set to the image median.
1373  # This group should include defects and bad amplifiers. As the
1374  # area covered by these defects are large, there's little
1375  # reason to expect that interpolation would provide a more
1376  # useful value.
1377  #
1378  # Smaller defects can be safely interpolated after the larger
1379  # regions have had their pixel values reset. This ensures
1380  # that the remaining defects adjacent to bad amplifiers (as an
1381  # example) do not attempt to interpolate extreme values.
1382  if self.config.doSetBadRegions:
1383  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1384  if badPixelCount > 0:
1385  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1386 
1387  if self.config.doInterpolate:
1388  self.log.info("Interpolating masked pixels.")
1389  isrFunctions.interpolateFromMask(
1390  maskedImage=ccdExposure.getMaskedImage(),
1391  fwhm=self.config.fwhm,
1392  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1393  maskNameList=list(self.config.maskListToInterpolate)
1394  )
1395 
1396  self.roughZeroPoint(ccdExposure)
1397 
1398  if self.config.doMeasureBackground:
1399  self.log.info("Measuring background level.")
1400  self.measureBackground(ccdExposure, self.config.qa)
1401 
1402  if self.config.qa is not None and self.config.qa.saveStats is True:
1403  for amp in ccd:
1404  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1405  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1406  afwMath.MEDIAN | afwMath.STDEVCLIP)
1407  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1408  qaStats.getValue(afwMath.MEDIAN))
1409  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1410  qaStats.getValue(afwMath.STDEVCLIP))
1411  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1412  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1413  qaStats.getValue(afwMath.STDEVCLIP))
1414 
1415  self.debugView(ccdExposure, "postISRCCD")
1416 
1417  return pipeBase.Struct(
1418  exposure=ccdExposure,
1419  ossThumb=ossThumb,
1420  flattenedThumb=flattenedThumb,
1421 
1422  preInterpolatedExposure=preInterpExp,
1423  outputExposure=ccdExposure,
1424  outputOssThumbnail=ossThumb,
1425  outputFlattenedThumbnail=flattenedThumb,
1426  )
1427 
1428  @pipeBase.timeMethod
1429  def runDataRef(self, sensorRef):
1430  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1431 
1432  This method contains the `CmdLineTask` interface to the ISR
1433  processing. All IO is handled here, freeing the `run()` method
1434  to manage only pixel-level calculations. The steps performed
1435  are:
1436  - Read in necessary detrending/isr/calibration data.
1437  - Process raw exposure in `run()`.
1438  - Persist the ISR-corrected exposure as "postISRCCD" if
1439  config.doWrite=True.
1440 
1441  Parameters
1442  ----------
1443  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1444  DataRef of the detector data to be processed
1445 
1446  Returns
1447  -------
1448  result : `lsst.pipe.base.Struct`
1449  Result struct with component:
1450  - ``exposure`` : `afw.image.Exposure`
1451  The fully ISR corrected exposure.
1452 
1453  Raises
1454  ------
1455  RuntimeError
1456  Raised if a configuration option is set to True, but the
1457  required calibration data does not exist.
1458 
1459  """
1460  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1461 
1462  ccdExposure = sensorRef.get(self.config.datasetType)
1463 
1464  camera = sensorRef.get("camera")
1465  isrData = self.readIsrData(sensorRef, ccdExposure)
1466 
1467  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1468 
1469  if self.config.doWrite:
1470  sensorRef.put(result.exposure, "postISRCCD")
1471  if result.preInterpolatedExposure is not None:
1472  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1473  if result.ossThumb is not None:
1474  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1475  if result.flattenedThumb is not None:
1476  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1477 
1478  return result
1479 
1480  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1481  """!Retrieve a calibration dataset for removing instrument signature.
1482 
1483  Parameters
1484  ----------
1485 
1486  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1487  DataRef of the detector data to find calibration datasets
1488  for.
1489  datasetType : `str`
1490  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1491  immediate : `Bool`
1492  If True, disable butler proxies to enable error handling
1493  within this routine.
1494 
1495  Returns
1496  -------
1497  exposure : `lsst.afw.image.Exposure`
1498  Requested calibration frame.
1499 
1500  Raises
1501  ------
1502  RuntimeError
1503  Raised if no matching calibration frame can be found.
1504  """
1505  try:
1506  exp = dataRef.get(datasetType, immediate=immediate)
1507  except Exception as exc1:
1508  if not self.config.fallbackFilterName:
1509  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1510  try:
1511  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1512  except Exception as exc2:
1513  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1514  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1515  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1516 
1517  if self.config.doAssembleIsrExposures:
1518  exp = self.assembleCcd.assembleCcd(exp)
1519  return exp
1520 
1521  def ensureExposure(self, inputExp, camera, detectorNum):
1522  """Ensure that the data returned by Butler is a fully constructed exposure.
1523 
1524  ISR requires exposure-level image data for historical reasons, so if we did
1525  not recieve that from Butler, construct it from what we have, modifying the
1526  input in place.
1527 
1528  Parameters
1529  ----------
1530  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1531  `lsst.afw.image.ImageF`
1532  The input data structure obtained from Butler.
1533  camera : `lsst.afw.cameraGeom.camera`
1534  The camera associated with the image. Used to find the appropriate
1535  detector.
1536  detectorNum : `int`
1537  The detector this exposure should match.
1538 
1539  Returns
1540  -------
1541  inputExp : `lsst.afw.image.Exposure`
1542  The re-constructed exposure, with appropriate detector parameters.
1543 
1544  Raises
1545  ------
1546  TypeError
1547  Raised if the input data cannot be used to construct an exposure.
1548  """
1549  if isinstance(inputExp, afwImage.DecoratedImageU):
1550  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1551  elif isinstance(inputExp, afwImage.ImageF):
1552  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1553  elif isinstance(inputExp, afwImage.MaskedImageF):
1554  inputExp = afwImage.makeExposure(inputExp)
1555  elif isinstance(inputExp, afwImage.Exposure):
1556  pass
1557  elif inputExp is None:
1558  # Assume this will be caught by the setup if it is a problem.
1559  return inputExp
1560  else:
1561  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1562  (type(inputExp), ))
1563 
1564  if inputExp.getDetector() is None:
1565  inputExp.setDetector(camera[detectorNum])
1566 
1567  return inputExp
1568 
1569  def convertIntToFloat(self, exposure):
1570  """Convert exposure image from uint16 to float.
1571 
1572  If the exposure does not need to be converted, the input is
1573  immediately returned. For exposures that are converted to use
1574  floating point pixels, the variance is set to unity and the
1575  mask to zero.
1576 
1577  Parameters
1578  ----------
1579  exposure : `lsst.afw.image.Exposure`
1580  The raw exposure to be converted.
1581 
1582  Returns
1583  -------
1584  newexposure : `lsst.afw.image.Exposure`
1585  The input ``exposure``, converted to floating point pixels.
1586 
1587  Raises
1588  ------
1589  RuntimeError
1590  Raised if the exposure type cannot be converted to float.
1591 
1592  """
1593  if isinstance(exposure, afwImage.ExposureF):
1594  # Nothing to be done
1595  self.log.debug("Exposure already of type float.")
1596  return exposure
1597  if not hasattr(exposure, "convertF"):
1598  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1599 
1600  newexposure = exposure.convertF()
1601  newexposure.variance[:] = 1
1602  newexposure.mask[:] = 0x0
1603 
1604  return newexposure
1605 
1606  def maskAmplifier(self, ccdExposure, amp, defects):
1607  """Identify bad amplifiers, saturated and suspect pixels.
1608 
1609  Parameters
1610  ----------
1611  ccdExposure : `lsst.afw.image.Exposure`
1612  Input exposure to be masked.
1613  amp : `lsst.afw.table.AmpInfoCatalog`
1614  Catalog of parameters defining the amplifier on this
1615  exposure to mask.
1616  defects : `lsst.meas.algorithms.Defects`
1617  List of defects. Used to determine if the entire
1618  amplifier is bad.
1619 
1620  Returns
1621  -------
1622  badAmp : `Bool`
1623  If this is true, the entire amplifier area is covered by
1624  defects and unusable.
1625 
1626  """
1627  maskedImage = ccdExposure.getMaskedImage()
1628 
1629  badAmp = False
1630 
1631  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1632  # comparison with current defects definition.
1633  if defects is not None:
1634  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1635 
1636  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1637  # association with pixels in current ccdExposure).
1638  if badAmp:
1639  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1640  afwImage.PARENT)
1641  maskView = dataView.getMask()
1642  maskView |= maskView.getPlaneBitMask("BAD")
1643  del maskView
1644  return badAmp
1645 
1646  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1647  # Saturation and suspect pixels can be masked now, though.
1648  limits = dict()
1649  if self.config.doSaturation and not badAmp:
1650  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1651  if self.config.doSuspect and not badAmp:
1652  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1653  if math.isfinite(self.config.saturation):
1654  limits.update({self.config.saturatedMaskName: self.config.saturation})
1655 
1656  for maskName, maskThreshold in limits.items():
1657  if not math.isnan(maskThreshold):
1658  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1659  isrFunctions.makeThresholdMask(
1660  maskedImage=dataView,
1661  threshold=maskThreshold,
1662  growFootprints=0,
1663  maskName=maskName
1664  )
1665 
1666  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1667  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1668  afwImage.PARENT)
1669  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1670  self.config.suspectMaskName])
1671  if numpy.all(maskView.getArray() & maskVal > 0):
1672  badAmp = True
1673  maskView |= maskView.getPlaneBitMask("BAD")
1674 
1675  return badAmp
1676 
1677  def overscanCorrection(self, ccdExposure, amp):
1678  """Apply overscan correction in place.
1679 
1680  This method does initial pixel rejection of the overscan
1681  region. The overscan can also be optionally segmented to
1682  allow for discontinuous overscan responses to be fit
1683  separately. The actual overscan subtraction is performed by
1684  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1685  which is called here after the amplifier is preprocessed.
1686 
1687  Parameters
1688  ----------
1689  ccdExposure : `lsst.afw.image.Exposure`
1690  Exposure to have overscan correction performed.
1691  amp : `lsst.afw.table.AmpInfoCatalog`
1692  The amplifier to consider while correcting the overscan.
1693 
1694  Returns
1695  -------
1696  overscanResults : `lsst.pipe.base.Struct`
1697  Result struct with components:
1698  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1699  Value or fit subtracted from the amplifier image data.
1700  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1701  Value or fit subtracted from the overscan image data.
1702  - ``overscanImage`` : `lsst.afw.image.Image`
1703  Image of the overscan region with the overscan
1704  correction applied. This quantity is used to estimate
1705  the amplifier read noise empirically.
1706 
1707  Raises
1708  ------
1709  RuntimeError
1710  Raised if the ``amp`` does not contain raw pixel information.
1711 
1712  See Also
1713  --------
1714  lsst.ip.isr.isrFunctions.overscanCorrection
1715  """
1716  if not amp.getHasRawInfo():
1717  raise RuntimeError("This method must be executed on an amp with raw information.")
1718 
1719  if amp.getRawHorizontalOverscanBBox().isEmpty():
1720  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1721  return None
1722 
1723  statControl = afwMath.StatisticsControl()
1724  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1725 
1726  # Determine the bounding boxes
1727  dataBBox = amp.getRawDataBBox()
1728  oscanBBox = amp.getRawHorizontalOverscanBBox()
1729  dx0 = 0
1730  dx1 = 0
1731 
1732  prescanBBox = amp.getRawPrescanBBox()
1733  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1734  dx0 += self.config.overscanNumLeadingColumnsToSkip
1735  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1736  else:
1737  dx0 += self.config.overscanNumTrailingColumnsToSkip
1738  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1739 
1740  # Determine if we need to work on subregions of the amplifier and overscan.
1741  imageBBoxes = []
1742  overscanBBoxes = []
1743 
1744  if ((self.config.overscanBiasJump and
1745  self.config.overscanBiasJumpLocation) and
1746  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1747  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1748  self.config.overscanBiasJumpDevices)):
1749  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1750  yLower = self.config.overscanBiasJumpLocation
1751  yUpper = dataBBox.getHeight() - yLower
1752  else:
1753  yUpper = self.config.overscanBiasJumpLocation
1754  yLower = dataBBox.getHeight() - yUpper
1755 
1756  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1757  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1758  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1759  lsst.geom.Extent2I(dx0, 0),
1760  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1761  yLower)))
1762 
1763  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1764  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1765  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1766  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1767  yUpper)))
1768  else:
1769  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1770  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1771  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1772  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1773  oscanBBox.getHeight())))
1774 
1775  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1776  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1777  ampImage = ccdExposure.maskedImage[imageBBox]
1778  overscanImage = ccdExposure.maskedImage[overscanBBox]
1779 
1780  overscanArray = overscanImage.image.array
1781  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1782  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1783  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1784 
1785  statControl = afwMath.StatisticsControl()
1786  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1787 
1788  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1789  overscanImage=overscanImage,
1790  fitType=self.config.overscanFitType,
1791  order=self.config.overscanOrder,
1792  collapseRej=self.config.overscanNumSigmaClip,
1793  statControl=statControl,
1794  overscanIsInt=self.config.overscanIsInt
1795  )
1796 
1797  # Measure average overscan levels and record them in the metadata.
1798  levelStat = afwMath.MEDIAN
1799  sigmaStat = afwMath.STDEVCLIP
1800 
1801  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1802  self.config.qa.flatness.nIter)
1803  metadata = ccdExposure.getMetadata()
1804  ampNum = amp.getName()
1805  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1806  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1807  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1808  else:
1809  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1810  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1811  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1812 
1813  return overscanResults
1814 
1815  def updateVariance(self, ampExposure, amp, overscanImage=None):
1816  """Set the variance plane using the amplifier gain and read noise
1817 
1818  The read noise is calculated from the ``overscanImage`` if the
1819  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1820  the value from the amplifier data is used.
1821 
1822  Parameters
1823  ----------
1824  ampExposure : `lsst.afw.image.Exposure`
1825  Exposure to process.
1826  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1827  Amplifier detector data.
1828  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1829  Image of overscan, required only for empirical read noise.
1830 
1831  See also
1832  --------
1833  lsst.ip.isr.isrFunctions.updateVariance
1834  """
1835  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1836  gain = amp.getGain()
1837 
1838  if math.isnan(gain):
1839  gain = 1.0
1840  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1841  elif gain <= 0:
1842  patchedGain = 1.0
1843  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1844  amp.getName(), gain, patchedGain)
1845  gain = patchedGain
1846 
1847  if self.config.doEmpiricalReadNoise and overscanImage is None:
1848  self.log.info("Overscan is none for EmpiricalReadNoise.")
1849 
1850  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1851  stats = afwMath.StatisticsControl()
1852  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1853  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1854  self.log.info("Calculated empirical read noise for amp %s: %f.",
1855  amp.getName(), readNoise)
1856  else:
1857  readNoise = amp.getReadNoise()
1858 
1859  isrFunctions.updateVariance(
1860  maskedImage=ampExposure.getMaskedImage(),
1861  gain=gain,
1862  readNoise=readNoise,
1863  )
1864 
1865  def darkCorrection(self, exposure, darkExposure, invert=False):
1866  """!Apply dark correction in place.
1867 
1868  Parameters
1869  ----------
1870  exposure : `lsst.afw.image.Exposure`
1871  Exposure to process.
1872  darkExposure : `lsst.afw.image.Exposure`
1873  Dark exposure of the same size as ``exposure``.
1874  invert : `Bool`, optional
1875  If True, re-add the dark to an already corrected image.
1876 
1877  Raises
1878  ------
1879  RuntimeError
1880  Raised if either ``exposure`` or ``darkExposure`` do not
1881  have their dark time defined.
1882 
1883  See Also
1884  --------
1885  lsst.ip.isr.isrFunctions.darkCorrection
1886  """
1887  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1888  if math.isnan(expScale):
1889  raise RuntimeError("Exposure darktime is NAN.")
1890  if darkExposure.getInfo().getVisitInfo() is not None:
1891  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1892  else:
1893  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1894  # so getDarkTime() does not exist.
1895  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1896  darkScale = 1.0
1897 
1898  if math.isnan(darkScale):
1899  raise RuntimeError("Dark calib darktime is NAN.")
1900  isrFunctions.darkCorrection(
1901  maskedImage=exposure.getMaskedImage(),
1902  darkMaskedImage=darkExposure.getMaskedImage(),
1903  expScale=expScale,
1904  darkScale=darkScale,
1905  invert=invert,
1906  trimToFit=self.config.doTrimToMatchCalib
1907  )
1908 
1909  def doLinearize(self, detector):
1910  """!Check if linearization is needed for the detector cameraGeom.
1911 
1912  Checks config.doLinearize and the linearity type of the first
1913  amplifier.
1914 
1915  Parameters
1916  ----------
1917  detector : `lsst.afw.cameraGeom.Detector`
1918  Detector to get linearity type from.
1919 
1920  Returns
1921  -------
1922  doLinearize : `Bool`
1923  If True, linearization should be performed.
1924  """
1925  return self.config.doLinearize and \
1926  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1927 
1928  def flatCorrection(self, exposure, flatExposure, invert=False):
1929  """!Apply flat correction in place.
1930 
1931  Parameters
1932  ----------
1933  exposure : `lsst.afw.image.Exposure`
1934  Exposure to process.
1935  flatExposure : `lsst.afw.image.Exposure`
1936  Flat exposure of the same size as ``exposure``.
1937  invert : `Bool`, optional
1938  If True, unflatten an already flattened image.
1939 
1940  See Also
1941  --------
1942  lsst.ip.isr.isrFunctions.flatCorrection
1943  """
1944  isrFunctions.flatCorrection(
1945  maskedImage=exposure.getMaskedImage(),
1946  flatMaskedImage=flatExposure.getMaskedImage(),
1947  scalingType=self.config.flatScalingType,
1948  userScale=self.config.flatUserScale,
1949  invert=invert,
1950  trimToFit=self.config.doTrimToMatchCalib
1951  )
1952 
1953  def saturationDetection(self, exposure, amp):
1954  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1955 
1956  Parameters
1957  ----------
1958  exposure : `lsst.afw.image.Exposure`
1959  Exposure to process. Only the amplifier DataSec is processed.
1960  amp : `lsst.afw.table.AmpInfoCatalog`
1961  Amplifier detector data.
1962 
1963  See Also
1964  --------
1965  lsst.ip.isr.isrFunctions.makeThresholdMask
1966  """
1967  if not math.isnan(amp.getSaturation()):
1968  maskedImage = exposure.getMaskedImage()
1969  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1970  isrFunctions.makeThresholdMask(
1971  maskedImage=dataView,
1972  threshold=amp.getSaturation(),
1973  growFootprints=0,
1974  maskName=self.config.saturatedMaskName,
1975  )
1976 
1977  def saturationInterpolation(self, exposure):
1978  """!Interpolate over saturated pixels, in place.
1979 
1980  This method should be called after `saturationDetection`, to
1981  ensure that the saturated pixels have been identified in the
1982  SAT mask. It should also be called after `assembleCcd`, since
1983  saturated regions may cross amplifier boundaries.
1984 
1985  Parameters
1986  ----------
1987  exposure : `lsst.afw.image.Exposure`
1988  Exposure to process.
1989 
1990  See Also
1991  --------
1992  lsst.ip.isr.isrTask.saturationDetection
1993  lsst.ip.isr.isrFunctions.interpolateFromMask
1994  """
1995  isrFunctions.interpolateFromMask(
1996  maskedImage=exposure.getMaskedImage(),
1997  fwhm=self.config.fwhm,
1998  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1999  maskNameList=list(self.config.saturatedMaskName),
2000  )
2001 
2002  def suspectDetection(self, exposure, amp):
2003  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2004 
2005  Parameters
2006  ----------
2007  exposure : `lsst.afw.image.Exposure`
2008  Exposure to process. Only the amplifier DataSec is processed.
2009  amp : `lsst.afw.table.AmpInfoCatalog`
2010  Amplifier detector data.
2011 
2012  See Also
2013  --------
2014  lsst.ip.isr.isrFunctions.makeThresholdMask
2015 
2016  Notes
2017  -----
2018  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2019  This is intended to indicate pixels that may be affected by unknown systematics;
2020  for example if non-linearity corrections above a certain level are unstable
2021  then that would be a useful value for suspectLevel. A value of `nan` indicates
2022  that no such level exists and no pixels are to be masked as suspicious.
2023  """
2024  suspectLevel = amp.getSuspectLevel()
2025  if math.isnan(suspectLevel):
2026  return
2027 
2028  maskedImage = exposure.getMaskedImage()
2029  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2030  isrFunctions.makeThresholdMask(
2031  maskedImage=dataView,
2032  threshold=suspectLevel,
2033  growFootprints=0,
2034  maskName=self.config.suspectMaskName,
2035  )
2036 
2037  def maskDefect(self, exposure, defectBaseList):
2038  """!Mask defects using mask plane "BAD", in place.
2039 
2040  Parameters
2041  ----------
2042  exposure : `lsst.afw.image.Exposure`
2043  Exposure to process.
2044  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2045  `lsst.afw.image.DefectBase`.
2046  List of defects to mask.
2047 
2048  Notes
2049  -----
2050  Call this after CCD assembly, since defects may cross amplifier boundaries.
2051  """
2052  maskedImage = exposure.getMaskedImage()
2053  if not isinstance(defectBaseList, Defects):
2054  # Promotes DefectBase to Defect
2055  defectList = Defects(defectBaseList)
2056  else:
2057  defectList = defectBaseList
2058  defectList.maskPixels(maskedImage, maskName="BAD")
2059 
2060  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2061  """!Mask edge pixels with applicable mask plane.
2062 
2063  Parameters
2064  ----------
2065  exposure : `lsst.afw.image.Exposure`
2066  Exposure to process.
2067  numEdgePixels : `int`, optional
2068  Number of edge pixels to mask.
2069  maskPlane : `str`, optional
2070  Mask plane name to use.
2071  """
2072  maskedImage = exposure.getMaskedImage()
2073  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2074 
2075  if numEdgePixels > 0:
2076  goodBBox = maskedImage.getBBox()
2077  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2078  goodBBox.grow(-numEdgePixels)
2079  # Mask pixels outside goodBBox
2080  SourceDetectionTask.setEdgeBits(
2081  maskedImage,
2082  goodBBox,
2083  maskBitMask
2084  )
2085 
2086  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2087  """Mask and interpolate defects using mask plane "BAD", in place.
2088 
2089  Parameters
2090  ----------
2091  exposure : `lsst.afw.image.Exposure`
2092  Exposure to process.
2093  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2094  `lsst.afw.image.DefectBase`.
2095  List of defects to mask and interpolate.
2096 
2097  See Also
2098  --------
2099  lsst.ip.isr.isrTask.maskDefect()
2100  """
2101  self.maskDefect(exposure, defectBaseList)
2102  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2103  maskPlane="SUSPECT")
2104  isrFunctions.interpolateFromMask(
2105  maskedImage=exposure.getMaskedImage(),
2106  fwhm=self.config.fwhm,
2107  growSaturatedFootprints=0,
2108  maskNameList=["BAD"],
2109  )
2110 
2111  def maskNan(self, exposure):
2112  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2113 
2114  Parameters
2115  ----------
2116  exposure : `lsst.afw.image.Exposure`
2117  Exposure to process.
2118 
2119  Notes
2120  -----
2121  We mask over all NaNs, including those that are masked with
2122  other bits (because those may or may not be interpolated over
2123  later, and we want to remove all NaNs). Despite this
2124  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2125  the historical name.
2126  """
2127  maskedImage = exposure.getMaskedImage()
2128 
2129  # Find and mask NaNs
2130  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2131  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2132  numNans = maskNans(maskedImage, maskVal)
2133  self.metadata.set("NUMNANS", numNans)
2134  if numNans > 0:
2135  self.log.warn("There were %d unmasked NaNs.", numNans)
2136 
2137  def maskAndInterpolateNan(self, exposure):
2138  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2139 
2140  Parameters
2141  ----------
2142  exposure : `lsst.afw.image.Exposure`
2143  Exposure to process.
2144 
2145  See Also
2146  --------
2147  lsst.ip.isr.isrTask.maskNan()
2148  """
2149  self.maskNan(exposure)
2150  isrFunctions.interpolateFromMask(
2151  maskedImage=exposure.getMaskedImage(),
2152  fwhm=self.config.fwhm,
2153  growSaturatedFootprints=0,
2154  maskNameList=["UNMASKEDNAN"],
2155  )
2156 
2157  def measureBackground(self, exposure, IsrQaConfig=None):
2158  """Measure the image background in subgrids, for quality control purposes.
2159 
2160  Parameters
2161  ----------
2162  exposure : `lsst.afw.image.Exposure`
2163  Exposure to process.
2164  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2165  Configuration object containing parameters on which background
2166  statistics and subgrids to use.
2167  """
2168  if IsrQaConfig is not None:
2169  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2170  IsrQaConfig.flatness.nIter)
2171  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2172  statsControl.setAndMask(maskVal)
2173  maskedImage = exposure.getMaskedImage()
2174  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2175  skyLevel = stats.getValue(afwMath.MEDIAN)
2176  skySigma = stats.getValue(afwMath.STDEVCLIP)
2177  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2178  metadata = exposure.getMetadata()
2179  metadata.set('SKYLEVEL', skyLevel)
2180  metadata.set('SKYSIGMA', skySigma)
2181 
2182  # calcluating flatlevel over the subgrids
2183  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2184  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2185  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2186  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2187  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2188  skyLevels = numpy.zeros((nX, nY))
2189 
2190  for j in range(nY):
2191  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2192  for i in range(nX):
2193  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2194 
2195  xLLC = xc - meshXHalf
2196  yLLC = yc - meshYHalf
2197  xURC = xc + meshXHalf - 1
2198  yURC = yc + meshYHalf - 1
2199 
2200  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2201  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2202 
2203  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2204 
2205  good = numpy.where(numpy.isfinite(skyLevels))
2206  skyMedian = numpy.median(skyLevels[good])
2207  flatness = (skyLevels[good] - skyMedian) / skyMedian
2208  flatness_rms = numpy.std(flatness)
2209  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2210 
2211  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2212  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2213  nX, nY, flatness_pp, flatness_rms)
2214 
2215  metadata.set('FLATNESS_PP', float(flatness_pp))
2216  metadata.set('FLATNESS_RMS', float(flatness_rms))
2217  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2218  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2219  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2220 
2221  def roughZeroPoint(self, exposure):
2222  """Set an approximate magnitude zero point for the exposure.
2223 
2224  Parameters
2225  ----------
2226  exposure : `lsst.afw.image.Exposure`
2227  Exposure to process.
2228  """
2229  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2230  if filterName in self.config.fluxMag0T1:
2231  fluxMag0 = self.config.fluxMag0T1[filterName]
2232  else:
2233  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2234  fluxMag0 = self.config.defaultFluxMag0T1
2235 
2236  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2237  if not expTime > 0: # handle NaN as well as <= 0
2238  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2239  return
2240 
2241  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2242  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2243 
2244  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2245  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2246 
2247  Parameters
2248  ----------
2249  ccdExposure : `lsst.afw.image.Exposure`
2250  Exposure to process.
2251  fpPolygon : `lsst.afw.geom.Polygon`
2252  Polygon in focal plane coordinates.
2253  """
2254  # Get ccd corners in focal plane coordinates
2255  ccd = ccdExposure.getDetector()
2256  fpCorners = ccd.getCorners(FOCAL_PLANE)
2257  ccdPolygon = Polygon(fpCorners)
2258 
2259  # Get intersection of ccd corners with fpPolygon
2260  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2261 
2262  # Transform back to pixel positions and build new polygon
2263  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2264  validPolygon = Polygon(ccdPoints)
2265  ccdExposure.getInfo().setValidPolygon(validPolygon)
2266 
2267  @contextmanager
2268  def flatContext(self, exp, flat, dark=None):
2269  """Context manager that applies and removes flats and darks,
2270  if the task is configured to apply them.
2271 
2272  Parameters
2273  ----------
2274  exp : `lsst.afw.image.Exposure`
2275  Exposure to process.
2276  flat : `lsst.afw.image.Exposure`
2277  Flat exposure the same size as ``exp``.
2278  dark : `lsst.afw.image.Exposure`, optional
2279  Dark exposure the same size as ``exp``.
2280 
2281  Yields
2282  ------
2283  exp : `lsst.afw.image.Exposure`
2284  The flat and dark corrected exposure.
2285  """
2286  if self.config.doDark and dark is not None:
2287  self.darkCorrection(exp, dark)
2288  if self.config.doFlat:
2289  self.flatCorrection(exp, flat)
2290  try:
2291  yield exp
2292  finally:
2293  if self.config.doFlat:
2294  self.flatCorrection(exp, flat, invert=True)
2295  if self.config.doDark and dark is not None:
2296  self.darkCorrection(exp, dark, invert=True)
2297 
2298  def debugView(self, exposure, stepname):
2299  """Utility function to examine ISR exposure at different stages.
2300 
2301  Parameters
2302  ----------
2303  exposure : `lsst.afw.image.Exposure`
2304  Exposure to view.
2305  stepname : `str`
2306  State of processing to view.
2307  """
2308  frame = getDebugFrame(self._display, stepname)
2309  if frame:
2310  display = getDisplay(frame)
2311  display.scale('asinh', 'zscale')
2312  display.mtv(exposure)
2313  prompt = "Press Enter to continue [c]... "
2314  while True:
2315  ans = input(prompt).lower()
2316  if ans in ("", "c",):
2317  break
2318 
2319 
2320 class FakeAmp(object):
2321  """A Detector-like object that supports returning gain and saturation level
2322 
2323  This is used when the input exposure does not have a detector.
2324 
2325  Parameters
2326  ----------
2327  exposure : `lsst.afw.image.Exposure`
2328  Exposure to generate a fake amplifier for.
2329  config : `lsst.ip.isr.isrTaskConfig`
2330  Configuration to apply to the fake amplifier.
2331  """
2332 
2333  def __init__(self, exposure, config):
2334  self._bbox = exposure.getBBox(afwImage.LOCAL)
2336  self._gain = config.gain
2337  self._readNoise = config.readNoise
2338  self._saturation = config.saturation
2339 
2340  def getBBox(self):
2341  return self._bbox
2342 
2343  def getRawBBox(self):
2344  return self._bbox
2345 
2346  def getHasRawInfo(self):
2347  return True # but see getRawHorizontalOverscanBBox()
2348 
2350  return self._RawHorizontalOverscanBBox
2351 
2352  def getGain(self):
2353  return self._gain
2354 
2355  def getReadNoise(self):
2356  return self._readNoise
2357 
2358  def getSaturation(self):
2359  return self._saturation
2360 
2361  def getSuspectLevel(self):
2362  return float("NaN")
2363 
2364 
2365 class RunIsrConfig(pexConfig.Config):
2366  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2367 
2368 
2369 class RunIsrTask(pipeBase.CmdLineTask):
2370  """Task to wrap the default IsrTask to allow it to be retargeted.
2371 
2372  The standard IsrTask can be called directly from a command line
2373  program, but doing so removes the ability of the task to be
2374  retargeted. As most cameras override some set of the IsrTask
2375  methods, this would remove those data-specific methods in the
2376  output post-ISR images. This wrapping class fixes the issue,
2377  allowing identical post-ISR images to be generated by both the
2378  processCcd and isrTask code.
2379  """
2380  ConfigClass = RunIsrConfig
2381  _DefaultName = "runIsr"
2382 
2383  def __init__(self, *args, **kwargs):
2384  super().__init__(*args, **kwargs)
2385  self.makeSubtask("isr")
2386 
2387  def runDataRef(self, dataRef):
2388  """
2389  Parameters
2390  ----------
2391  dataRef : `lsst.daf.persistence.ButlerDataRef`
2392  data reference of the detector data to be processed
2393 
2394  Returns
2395  -------
2396  result : `pipeBase.Struct`
2397  Result struct with component:
2398 
2399  - exposure : `lsst.afw.image.Exposure`
2400  Post-ISR processed exposure.
2401  """
2402  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1429
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2157
def debugView(self, exposure, stepname)
Definition: isrTask.py:2298
def __init__(self, kwargs)
Definition: isrTask.py:783
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1521
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:833
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2060
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:792
def runDataRef(self, dataRef)
Definition: isrTask.py:2387
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:981
def __init__(self, args, kwargs)
Definition: isrTask.py:2383
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2221
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2086
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2349
def maskNan(self, exposure)
Definition: isrTask.py:2111
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2037
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1677
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1569
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1928
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1480
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1865
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1909
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2244
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1606
def __init__(self, config=None)
Definition: isrTask.py:161
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2268
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1815
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2137
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2002
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1977
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1953
def __init__(self, exposure, config)
Definition: isrTask.py:2333