lsst.ip.isr  18.0.0-8-g697fdba+3
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["instrument"],
149  )
150  illumMaskedImage = pipeBase.InputDatasetField(
151  doc="Input illumination correction.",
152  name="illum",
153  scalar=True,
154  storageClass="MaskedImageF",
155  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
156  )
157 
158  # output datasets
159  outputExposure = pipeBase.OutputDatasetField(
160  doc="Output ISR processed exposure.",
161  name="postISRCCD",
162  scalar=True,
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = pipeBase.OutputDatasetField(
167  doc="Output Overscan-subtracted thumbnail image.",
168  name="OssThumb",
169  scalar=True,
170  storageClass="Thumbnail",
171  dimensions=["instrument", "visit", "detector"],
172  )
173  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
174  doc="Output flat-corrected thumbnail image.",
175  name="FlattenedThumb",
176  scalar=True,
177  storageClass="TextStorage",
178  dimensions=["instrument", "visit", "detector"],
179  )
180 
181  quantum = pipeBase.QuantumConfig(
182  dimensions=["visit", "detector", "instrument"],
183  )
184 
185 
186  datasetType = pexConfig.Field(
187  dtype=str,
188  doc="Dataset type for input data; users will typically leave this alone, "
189  "but camera-specific ISR tasks will override it",
190  default="raw",
191  )
192 
193  fallbackFilterName = pexConfig.Field(
194  dtype=str,
195  doc="Fallback default filter name for calibrations.",
196  optional=True
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
202  )
203  fwhm = pexConfig.Field(
204  dtype=float,
205  doc="FWHM of PSF in arcseconds.",
206  default=1.0,
207  )
208  qa = pexConfig.ConfigField(
209  dtype=isrQa.IsrQaConfig,
210  doc="QA related configuration options.",
211  )
212 
213  # Image conversion configuration
214  doConvertIntToFloat = pexConfig.Field(
215  dtype=bool,
216  doc="Convert integer raw images to floating point values?",
217  default=True,
218  )
219 
220  # Saturated pixel handling.
221  doSaturation = pexConfig.Field(
222  dtype=bool,
223  doc="Mask saturated pixels? NB: this is totally independent of the"
224  " interpolation option - this is ONLY setting the bits in the mask."
225  " To have them interpolated make sure doSaturationInterpolation=True",
226  default=True,
227  )
228  saturatedMaskName = pexConfig.Field(
229  dtype=str,
230  doc="Name of mask plane to use in saturation detection and interpolation",
231  default="SAT",
232  )
233  saturation = pexConfig.Field(
234  dtype=float,
235  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
236  default=float("NaN"),
237  )
238  growSaturationFootprintSize = pexConfig.Field(
239  dtype=int,
240  doc="Number of pixels by which to grow the saturation footprints",
241  default=1,
242  )
243 
244  # Suspect pixel handling.
245  doSuspect = pexConfig.Field(
246  dtype=bool,
247  doc="Mask suspect pixels?",
248  default=True,
249  )
250  suspectMaskName = pexConfig.Field(
251  dtype=str,
252  doc="Name of mask plane to use for suspect pixels",
253  default="SUSPECT",
254  )
255  numEdgeSuspect = pexConfig.Field(
256  dtype=int,
257  doc="Number of edge pixels to be flagged as untrustworthy.",
258  default=0,
259  )
260 
261  # Initial masking options.
262  doSetBadRegions = pexConfig.Field(
263  dtype=bool,
264  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
265  default=True,
266  )
267  badStatistic = pexConfig.ChoiceField(
268  dtype=str,
269  doc="How to estimate the average value for BAD regions.",
270  default='MEANCLIP',
271  allowed={
272  "MEANCLIP": "Correct using the (clipped) mean of good data",
273  "MEDIAN": "Correct using the median of the good data",
274  },
275  )
276 
277  # Overscan subtraction configuration.
278  doOverscan = pexConfig.Field(
279  dtype=bool,
280  doc="Do overscan subtraction?",
281  default=True,
282  )
283  overscanFitType = pexConfig.ChoiceField(
284  dtype=str,
285  doc="The method for fitting the overscan bias level.",
286  default='MEDIAN',
287  allowed={
288  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
289  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
290  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
291  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
292  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
293  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
294  "MEAN": "Correct using the mean of the overscan region",
295  "MEANCLIP": "Correct using a clipped mean of the overscan region",
296  "MEDIAN": "Correct using the median of the overscan region",
297  },
298  )
299  overscanOrder = pexConfig.Field(
300  dtype=int,
301  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
302  "or number of spline knots if overscan fit type is a spline."),
303  default=1,
304  )
305  overscanNumSigmaClip = pexConfig.Field(
306  dtype=float,
307  doc="Rejection threshold (sigma) for collapsing overscan before fit",
308  default=3.0,
309  )
310  overscanIsInt = pexConfig.Field(
311  dtype=bool,
312  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
313  default=True,
314  )
315  overscanNumLeadingColumnsToSkip = pexConfig.Field(
316  dtype=int,
317  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
318  default=0,
319  )
320  overscanNumTrailingColumnsToSkip = pexConfig.Field(
321  dtype=int,
322  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
323  default=0,
324  )
325  overscanMaxDev = pexConfig.Field(
326  dtype=float,
327  doc="Maximum deviation from the median for overscan",
328  default=1000.0, check=lambda x: x > 0
329  )
330  overscanBiasJump = pexConfig.Field(
331  dtype=bool,
332  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
333  default=False,
334  )
335  overscanBiasJumpKeyword = pexConfig.Field(
336  dtype=str,
337  doc="Header keyword containing information about devices.",
338  default="NO_SUCH_KEY",
339  )
340  overscanBiasJumpDevices = pexConfig.ListField(
341  dtype=str,
342  doc="List of devices that need piecewise overscan correction.",
343  default=(),
344  )
345  overscanBiasJumpLocation = pexConfig.Field(
346  dtype=int,
347  doc="Location of bias jump along y-axis.",
348  default=0,
349  )
350 
351  # Amplifier to CCD assembly configuration
352  doAssembleCcd = pexConfig.Field(
353  dtype=bool,
354  default=True,
355  doc="Assemble amp-level exposures into a ccd-level exposure?"
356  )
357  assembleCcd = pexConfig.ConfigurableField(
358  target=AssembleCcdTask,
359  doc="CCD assembly task",
360  )
361 
362  # General calibration configuration.
363  doAssembleIsrExposures = pexConfig.Field(
364  dtype=bool,
365  default=False,
366  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
367  )
368  doTrimToMatchCalib = pexConfig.Field(
369  dtype=bool,
370  default=False,
371  doc="Trim raw data to match calibration bounding boxes?"
372  )
373 
374  # Bias subtraction.
375  doBias = pexConfig.Field(
376  dtype=bool,
377  doc="Apply bias frame correction?",
378  default=True,
379  )
380  biasDataProductName = pexConfig.Field(
381  dtype=str,
382  doc="Name of the bias data product",
383  default="bias",
384  )
385 
386  # Variance construction
387  doVariance = pexConfig.Field(
388  dtype=bool,
389  doc="Calculate variance?",
390  default=True
391  )
392  gain = pexConfig.Field(
393  dtype=float,
394  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
395  default=float("NaN"),
396  )
397  readNoise = pexConfig.Field(
398  dtype=float,
399  doc="The read noise to use if no Detector is present in the Exposure",
400  default=0.0,
401  )
402  doEmpiricalReadNoise = pexConfig.Field(
403  dtype=bool,
404  default=False,
405  doc="Calculate empirical read noise instead of value from AmpInfo data?"
406  )
407 
408  # Linearization.
409  doLinearize = pexConfig.Field(
410  dtype=bool,
411  doc="Correct for nonlinearity of the detector's response?",
412  default=True,
413  )
414 
415  # Crosstalk.
416  doCrosstalk = pexConfig.Field(
417  dtype=bool,
418  doc="Apply intra-CCD crosstalk correction?",
419  default=False,
420  )
421  doCrosstalkBeforeAssemble = pexConfig.Field(
422  dtype=bool,
423  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
424  default=True,
425  )
426  crosstalk = pexConfig.ConfigurableField(
427  target=CrosstalkTask,
428  doc="Intra-CCD crosstalk correction",
429  )
430 
431  # Masking options.
432  doDefect = pexConfig.Field(
433  dtype=bool,
434  doc="Apply correction for CCD defects, e.g. hot pixels?",
435  default=True,
436  )
437  numEdgeSuspect = pexConfig.Field(
438  dtype=int,
439  doc="Number of edge pixels to be flagged as untrustworthy.",
440  default=0,
441  )
442  doNanMasking = pexConfig.Field(
443  dtype=bool,
444  doc="Mask NAN pixels?",
445  default=True,
446  )
447  doWidenSaturationTrails = pexConfig.Field(
448  dtype=bool,
449  doc="Widen bleed trails based on their width?",
450  default=True
451  )
452 
453  # Brighter-Fatter correction.
454  doBrighterFatter = pexConfig.Field(
455  dtype=bool,
456  default=False,
457  doc="Apply the brighter fatter correction"
458  )
459  brighterFatterLevel = pexConfig.ChoiceField(
460  dtype=str,
461  default="DETECTOR",
462  doc="The level at which to correct for brighter-fatter.",
463  allowed={
464  "AMP": "Every amplifier treated separately.",
465  "DETECTOR": "One kernel per detector",
466  }
467  )
468  brighterFatterKernelFile = pexConfig.Field(
469  dtype=str,
470  default='',
471  doc="Kernel file used for the brighter fatter correction"
472  )
473  brighterFatterMaxIter = pexConfig.Field(
474  dtype=int,
475  default=10,
476  doc="Maximum number of iterations for the brighter fatter correction"
477  )
478  brighterFatterThreshold = pexConfig.Field(
479  dtype=float,
480  default=1000,
481  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
482  " absolute value of the difference between the current corrected image and the one"
483  " from the previous iteration summed over all the pixels."
484  )
485  brighterFatterApplyGain = pexConfig.Field(
486  dtype=bool,
487  default=True,
488  doc="Should the gain be applied when applying the brighter fatter correction?"
489  )
490 
491  # Dark subtraction.
492  doDark = pexConfig.Field(
493  dtype=bool,
494  doc="Apply dark frame correction?",
495  default=True,
496  )
497  darkDataProductName = pexConfig.Field(
498  dtype=str,
499  doc="Name of the dark data product",
500  default="dark",
501  )
502 
503  # Camera-specific stray light removal.
504  doStrayLight = pexConfig.Field(
505  dtype=bool,
506  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
507  default=False,
508  )
509  strayLight = pexConfig.ConfigurableField(
510  target=StrayLightTask,
511  doc="y-band stray light correction"
512  )
513 
514  # Flat correction.
515  doFlat = pexConfig.Field(
516  dtype=bool,
517  doc="Apply flat field correction?",
518  default=True,
519  )
520  flatDataProductName = pexConfig.Field(
521  dtype=str,
522  doc="Name of the flat data product",
523  default="flat",
524  )
525  flatScalingType = pexConfig.ChoiceField(
526  dtype=str,
527  doc="The method for scaling the flat on the fly.",
528  default='USER',
529  allowed={
530  "USER": "Scale by flatUserScale",
531  "MEAN": "Scale by the inverse of the mean",
532  "MEDIAN": "Scale by the inverse of the median",
533  },
534  )
535  flatUserScale = pexConfig.Field(
536  dtype=float,
537  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
538  default=1.0,
539  )
540  doTweakFlat = pexConfig.Field(
541  dtype=bool,
542  doc="Tweak flats to match observed amplifier ratios?",
543  default=False
544  )
545 
546  # Amplifier normalization based on gains instead of using flats configuration.
547  doApplyGains = pexConfig.Field(
548  dtype=bool,
549  doc="Correct the amplifiers for their gains instead of applying flat correction",
550  default=False,
551  )
552  normalizeGains = pexConfig.Field(
553  dtype=bool,
554  doc="Normalize all the amplifiers in each CCD to have the same median value.",
555  default=False,
556  )
557 
558  # Fringe correction.
559  doFringe = pexConfig.Field(
560  dtype=bool,
561  doc="Apply fringe correction?",
562  default=True,
563  )
564  fringe = pexConfig.ConfigurableField(
565  target=FringeTask,
566  doc="Fringe subtraction task",
567  )
568  fringeAfterFlat = pexConfig.Field(
569  dtype=bool,
570  doc="Do fringe subtraction after flat-fielding?",
571  default=True,
572  )
573 
574  # Distortion model application.
575  doAddDistortionModel = pexConfig.Field(
576  dtype=bool,
577  doc="Apply a distortion model based on camera geometry to the WCS?",
578  default=True,
579  )
580 
581  # Initial CCD-level background statistics options.
582  doMeasureBackground = pexConfig.Field(
583  dtype=bool,
584  doc="Measure the background level on the reduced image?",
585  default=False,
586  )
587 
588  # Camera-specific masking configuration.
589  doCameraSpecificMasking = pexConfig.Field(
590  dtype=bool,
591  doc="Mask camera-specific bad regions?",
592  default=False,
593  )
594  masking = pexConfig.ConfigurableField(
595  target=MaskingTask,
596  doc="Masking task."
597  )
598 
599  # Interpolation options.
600 
601  doInterpolate = pexConfig.Field(
602  dtype=bool,
603  doc="Interpolate masked pixels?",
604  default=True,
605  )
606  doSaturationInterpolation = pexConfig.Field(
607  dtype=bool,
608  doc="Perform interpolation over pixels masked as saturated?"
609  " NB: This is independent of doSaturation; if that is False this plane"
610  " will likely be blank, resulting in a no-op here.",
611  default=True,
612  )
613  doNanInterpolation = pexConfig.Field(
614  dtype=bool,
615  doc="Perform interpolation over pixels masked as NaN?"
616  " NB: This is independent of doNanMasking; if that is False this plane"
617  " will likely be blank, resulting in a no-op here.",
618  default=True,
619  )
620  doNanInterpAfterFlat = pexConfig.Field(
621  dtype=bool,
622  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
623  "also have to interpolate them before flat-fielding."),
624  default=False,
625  )
626  maskListToInterpolate = pexConfig.ListField(
627  dtype=str,
628  doc="List of mask planes that should be interpolated.",
629  default=['SAT', 'BAD', 'UNMASKEDNAN'],
630  )
631  doSaveInterpPixels = pexConfig.Field(
632  dtype=bool,
633  doc="Save a copy of the pre-interpolated pixel values?",
634  default=False,
635  )
636 
637  # Default photometric calibration options.
638  fluxMag0T1 = pexConfig.DictField(
639  keytype=str,
640  itemtype=float,
641  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
642  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
643  ))
644  )
645  defaultFluxMag0T1 = pexConfig.Field(
646  dtype=float,
647  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
648  default=pow(10.0, 0.4*28.0)
649  )
650 
651  # Vignette correction configuration.
652  doVignette = pexConfig.Field(
653  dtype=bool,
654  doc="Apply vignetting parameters?",
655  default=False,
656  )
657  vignette = pexConfig.ConfigurableField(
658  target=VignetteTask,
659  doc="Vignetting task.",
660  )
661 
662  # Transmission curve configuration.
663  doAttachTransmissionCurve = pexConfig.Field(
664  dtype=bool,
665  default=False,
666  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
667  )
668  doUseOpticsTransmission = pexConfig.Field(
669  dtype=bool,
670  default=True,
671  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
672  )
673  doUseFilterTransmission = pexConfig.Field(
674  dtype=bool,
675  default=True,
676  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
677  )
678  doUseSensorTransmission = pexConfig.Field(
679  dtype=bool,
680  default=True,
681  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
682  )
683  doUseAtmosphereTransmission = pexConfig.Field(
684  dtype=bool,
685  default=True,
686  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
687  )
688 
689  # Illumination correction.
690  doIlluminationCorrection = pexConfig.Field(
691  dtype=bool,
692  default=False,
693  doc="Perform illumination correction?"
694  )
695  illuminationCorrectionDataProductName = pexConfig.Field(
696  dtype=str,
697  doc="Name of the illumination correction data product.",
698  default="illumcor",
699  )
700  illumScale = pexConfig.Field(
701  dtype=float,
702  doc="Scale factor for the illumination correction.",
703  default=1.0,
704  )
705  illumFilters = pexConfig.ListField(
706  dtype=str,
707  default=[],
708  doc="Only perform illumination correction for these filters."
709  )
710 
711  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
712  doWrite = pexConfig.Field(
713  dtype=bool,
714  doc="Persist postISRCCD?",
715  default=True,
716  )
717 
718  def validate(self):
719  super().validate()
720  if self.doFlat and self.doApplyGains:
721  raise ValueError("You may not specify both doFlat and doApplyGains")
722  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
723  self.config.maskListToInterpolate.append("SAT")
724  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
725  self.config.maskListToInterpolate.append("UNMASKEDNAN")
726 
727 
728 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
729  """Apply common instrument signature correction algorithms to a raw frame.
730 
731  The process for correcting imaging data is very similar from
732  camera to camera. This task provides a vanilla implementation of
733  doing these corrections, including the ability to turn certain
734  corrections off if they are not needed. The inputs to the primary
735  method, `run()`, are a raw exposure to be corrected and the
736  calibration data products. The raw input is a single chip sized
737  mosaic of all amps including overscans and other non-science
738  pixels. The method `runDataRef()` identifies and defines the
739  calibration data products, and is intended for use by a
740  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
741  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
742  subclassed for different camera, although the most camera specific
743  methods have been split into subtasks that can be redirected
744  appropriately.
745 
746  The __init__ method sets up the subtasks for ISR processing, using
747  the defaults from `lsst.ip.isr`.
748 
749  Parameters
750  ----------
751  args : `list`
752  Positional arguments passed to the Task constructor. None used at this time.
753  kwargs : `dict`, optional
754  Keyword arguments passed on to the Task constructor. None used at this time.
755  """
756  ConfigClass = IsrTaskConfig
757  _DefaultName = "isr"
758 
759  def __init__(self, **kwargs):
760  super().__init__(**kwargs)
761  self.makeSubtask("assembleCcd")
762  self.makeSubtask("crosstalk")
763  self.makeSubtask("strayLight")
764  self.makeSubtask("fringe")
765  self.makeSubtask("masking")
766  self.makeSubtask("vignette")
767 
768  @classmethod
769  def getInputDatasetTypes(cls, config):
770  inputTypeDict = super().getInputDatasetTypes(config)
771 
772  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
773  # need because the configuration tells us we will not be bothering with the
774  # correction that uses that IDT.
775  if config.doBias is not True:
776  inputTypeDict.pop("bias", None)
777  if config.doLinearize is not True:
778  inputTypeDict.pop("linearizer", None)
779  if config.doCrosstalk is not True:
780  inputTypeDict.pop("crosstalkSources", None)
781  if config.doBrighterFatter is not True:
782  inputTypeDict.pop("bfKernel", None)
783  if config.doDefect is not True:
784  inputTypeDict.pop("defects", None)
785  if config.doDark is not True:
786  inputTypeDict.pop("dark", None)
787  if config.doFlat is not True:
788  inputTypeDict.pop("flat", None)
789  if config.doAttachTransmissionCurve is not True:
790  inputTypeDict.pop("opticsTransmission", None)
791  inputTypeDict.pop("filterTransmission", None)
792  inputTypeDict.pop("sensorTransmission", None)
793  inputTypeDict.pop("atmosphereTransmission", None)
794  if config.doUseOpticsTransmission is not True:
795  inputTypeDict.pop("opticsTransmission", None)
796  if config.doUseFilterTransmission is not True:
797  inputTypeDict.pop("filterTransmission", None)
798  if config.doUseSensorTransmission is not True:
799  inputTypeDict.pop("sensorTransmission", None)
800  if config.doUseAtmosphereTransmission is not True:
801  inputTypeDict.pop("atmosphereTransmission", None)
802  if config.doIlluminationCorrection is not True:
803  inputTypeDict.pop("illumMaskedImage", None)
804 
805  return inputTypeDict
806 
807  @classmethod
808  def getOutputDatasetTypes(cls, config):
809  outputTypeDict = super().getOutputDatasetTypes(config)
810 
811  if config.qa.doThumbnailOss is not True:
812  outputTypeDict.pop("outputOssThumbnail", None)
813  if config.qa.doThumbnailFlattened is not True:
814  outputTypeDict.pop("outputFlattenedThumbnail", None)
815  if config.doWrite is not True:
816  outputTypeDict.pop("outputExposure", None)
817 
818  return outputTypeDict
819 
820  @classmethod
821  def getPrerequisiteDatasetTypes(cls, config):
822  # Input calibration datasets should not constrain the QuantumGraph
823  # (it'd be confusing if not having flats just silently resulted in no
824  # data being processed). Our nomenclature for that is that these are
825  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
826  names = set(cls.getInputDatasetTypes(config))
827  names.remove("ccdExposure")
828  return names
829 
830  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
831  try:
832  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
833  except Exception as e:
834  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
835  (inputDataIds, e))
836 
837  inputData['isGen3'] = True
838 
839  if self.config.doLinearize is True:
840  if 'linearizer' not in inputData.keys():
841  detector = inputData['camera'][inputData['detectorNum']]
842  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
843  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
844 
845  if inputData['defects'] is not None:
846  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
847  # masking expects a list of defects defined by their bounding box
848  if not isinstance(inputData["defects"], Defects):
849  inputData["defects"] = Defects.fromTable(inputData["defects"])
850 
851  # Broken: DM-17169
852  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
853  # fixed for non-HSC cameras in the future.
854  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
855  # if self.config.doCrosstalk else None)
856 
857  # Broken: DM-17152
858  # Fringes are not tested to be handled correctly by Gen3 butler.
859  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
860  # assembler=self.assembleCcd
861  # if self.config.doAssembleIsrExposures else None)
862  # if self.config.doFringe and
863  # self.fringe.checkFilter(inputData['ccdExposure'])
864  # else pipeBase.Struct(fringes=None))
865 
866  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
867 
868  def makeDatasetType(self, dsConfig):
869  return super().makeDatasetType(dsConfig)
870 
871  def readIsrData(self, dataRef, rawExposure):
872  """!Retrieve necessary frames for instrument signature removal.
873 
874  Pre-fetching all required ISR data products limits the IO
875  required by the ISR. Any conflict between the calibration data
876  available and that needed for ISR is also detected prior to
877  doing processing, allowing it to fail quickly.
878 
879  Parameters
880  ----------
881  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
882  Butler reference of the detector data to be processed
883  rawExposure : `afw.image.Exposure`
884  The raw exposure that will later be corrected with the
885  retrieved calibration data; should not be modified in this
886  method.
887 
888  Returns
889  -------
890  result : `lsst.pipe.base.Struct`
891  Result struct with components (which may be `None`):
892  - ``bias``: bias calibration frame (`afw.image.Exposure`)
893  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
894  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
895  - ``dark``: dark calibration frame (`afw.image.Exposure`)
896  - ``flat``: flat calibration frame (`afw.image.Exposure`)
897  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
898  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
899  - ``fringes``: `lsst.pipe.base.Struct` with components:
900  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
901  - ``seed``: random seed derived from the ccdExposureId for random
902  number generator (`uint32`).
903  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
904  A ``TransmissionCurve`` that represents the throughput of the optics,
905  to be evaluated in focal-plane coordinates.
906  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
907  A ``TransmissionCurve`` that represents the throughput of the filter
908  itself, to be evaluated in focal-plane coordinates.
909  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
910  A ``TransmissionCurve`` that represents the throughput of the sensor
911  itself, to be evaluated in post-assembly trimmed detector coordinates.
912  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
913  A ``TransmissionCurve`` that represents the throughput of the
914  atmosphere, assumed to be spatially constant.
915  - ``strayLightData`` : `object`
916  An opaque object containing calibration information for
917  stray-light correction. If `None`, no correction will be
918  performed.
919  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
920 
921  Raises
922  ------
923  NotImplementedError :
924  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
925  """
926  ccd = rawExposure.getDetector()
927  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
928  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
929  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
930  if self.config.doBias else None)
931  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
932  linearizer = (dataRef.get("linearizer", immediate=True)
933  if self.doLinearize(ccd) else None)
934  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
935  if self.config.doCrosstalk else None)
936  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
937  if self.config.doDark else None)
938  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
939  if self.config.doFlat else None)
940 
941  brighterFatterKernel = None
942  if self.config.doBrighterFatter is True:
943 
944  # Use the new-style cp_pipe version of the kernel is it exists.
945  try:
946  brighterFatterKernel = dataRef.get("brighterFatterKernel")
947  except NoResults:
948  # Fall back to the old-style numpy-ndarray style kernel if necessary.
949  try:
950  brighterFatterKernel = dataRef.get("bfKernel")
951  except NoResults:
952  brighterFatterKernel = None
953  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
954  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
955  # this detector, or raise an error.
956  if self.config.brighterFatterLevel == 'DETECTOR':
957  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
958  else:
959  # TODO DM-15631 for implementing this
960  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
961 
962  defectList = (dataRef.get("defects")
963  if self.config.doDefect else None)
964  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
965  if self.config.doAssembleIsrExposures else None)
966  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
967  else pipeBase.Struct(fringes=None))
968 
969  if self.config.doAttachTransmissionCurve:
970  opticsTransmission = (dataRef.get("transmission_optics")
971  if self.config.doUseOpticsTransmission else None)
972  filterTransmission = (dataRef.get("transmission_filter")
973  if self.config.doUseFilterTransmission else None)
974  sensorTransmission = (dataRef.get("transmission_sensor")
975  if self.config.doUseSensorTransmission else None)
976  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
977  if self.config.doUseAtmosphereTransmission else None)
978  else:
979  opticsTransmission = None
980  filterTransmission = None
981  sensorTransmission = None
982  atmosphereTransmission = None
983 
984  if self.config.doStrayLight:
985  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
986  else:
987  strayLightData = None
988 
989  illumMaskedImage = (self.getIsrExposure(dataRef,
990  self.config.illuminationCorrectionDataProductName).getMaskedImage()
991  if (self.config.doIlluminationCorrection and
992  filterName in self.config.illumFilters)
993  else None)
994 
995  # Struct should include only kwargs to run()
996  return pipeBase.Struct(bias=biasExposure,
997  linearizer=linearizer,
998  crosstalkSources=crosstalkSources,
999  dark=darkExposure,
1000  flat=flatExposure,
1001  bfKernel=brighterFatterKernel,
1002  defects=defectList,
1003  fringes=fringeStruct,
1004  opticsTransmission=opticsTransmission,
1005  filterTransmission=filterTransmission,
1006  sensorTransmission=sensorTransmission,
1007  atmosphereTransmission=atmosphereTransmission,
1008  strayLightData=strayLightData,
1009  illumMaskedImage=illumMaskedImage
1010  )
1011 
1012  @pipeBase.timeMethod
1013  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1014  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1015  opticsTransmission=None, filterTransmission=None,
1016  sensorTransmission=None, atmosphereTransmission=None,
1017  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1018  isGen3=False,
1019  ):
1020  """!Perform instrument signature removal on an exposure.
1021 
1022  Steps included in the ISR processing, in order performed, are:
1023  - saturation and suspect pixel masking
1024  - overscan subtraction
1025  - CCD assembly of individual amplifiers
1026  - bias subtraction
1027  - variance image construction
1028  - linearization of non-linear response
1029  - crosstalk masking
1030  - brighter-fatter correction
1031  - dark subtraction
1032  - fringe correction
1033  - stray light subtraction
1034  - flat correction
1035  - masking of known defects and camera specific features
1036  - vignette calculation
1037  - appending transmission curve and distortion model
1038 
1039  Parameters
1040  ----------
1041  ccdExposure : `lsst.afw.image.Exposure`
1042  The raw exposure that is to be run through ISR. The
1043  exposure is modified by this method.
1044  camera : `lsst.afw.cameraGeom.Camera`, optional
1045  The camera geometry for this exposure. Used to select the
1046  distortion model appropriate for this data.
1047  bias : `lsst.afw.image.Exposure`, optional
1048  Bias calibration frame.
1049  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1050  Functor for linearization.
1051  crosstalkSources : `list`, optional
1052  List of possible crosstalk sources.
1053  dark : `lsst.afw.image.Exposure`, optional
1054  Dark calibration frame.
1055  flat : `lsst.afw.image.Exposure`, optional
1056  Flat calibration frame.
1057  bfKernel : `numpy.ndarray`, optional
1058  Brighter-fatter kernel.
1059  defects : `lsst.meas.algorithms.Defects`, optional
1060  List of defects.
1061  fringes : `lsst.pipe.base.Struct`, optional
1062  Struct containing the fringe correction data, with
1063  elements:
1064  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1065  - ``seed``: random seed derived from the ccdExposureId for random
1066  number generator (`uint32`)
1067  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1068  A ``TransmissionCurve`` that represents the throughput of the optics,
1069  to be evaluated in focal-plane coordinates.
1070  filterTransmission : `lsst.afw.image.TransmissionCurve`
1071  A ``TransmissionCurve`` that represents the throughput of the filter
1072  itself, to be evaluated in focal-plane coordinates.
1073  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1074  A ``TransmissionCurve`` that represents the throughput of the sensor
1075  itself, to be evaluated in post-assembly trimmed detector coordinates.
1076  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1077  A ``TransmissionCurve`` that represents the throughput of the
1078  atmosphere, assumed to be spatially constant.
1079  detectorNum : `int`, optional
1080  The integer number for the detector to process.
1081  isGen3 : bool, optional
1082  Flag this call to run() as using the Gen3 butler environment.
1083  strayLightData : `object`, optional
1084  Opaque object containing calibration information for stray-light
1085  correction. If `None`, no correction will be performed.
1086  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1087  Illumination correction image.
1088 
1089  Returns
1090  -------
1091  result : `lsst.pipe.base.Struct`
1092  Result struct with component:
1093  - ``exposure`` : `afw.image.Exposure`
1094  The fully ISR corrected exposure.
1095  - ``outputExposure`` : `afw.image.Exposure`
1096  An alias for `exposure`
1097  - ``ossThumb`` : `numpy.ndarray`
1098  Thumbnail image of the exposure after overscan subtraction.
1099  - ``flattenedThumb`` : `numpy.ndarray`
1100  Thumbnail image of the exposure after flat-field correction.
1101 
1102  Raises
1103  ------
1104  RuntimeError
1105  Raised if a configuration option is set to True, but the
1106  required calibration data has not been specified.
1107 
1108  Notes
1109  -----
1110  The current processed exposure can be viewed by setting the
1111  appropriate lsstDebug entries in the `debug.display`
1112  dictionary. The names of these entries correspond to some of
1113  the IsrTaskConfig Boolean options, with the value denoting the
1114  frame to use. The exposure is shown inside the matching
1115  option check and after the processing of that step has
1116  finished. The steps with debug points are:
1117 
1118  doAssembleCcd
1119  doBias
1120  doCrosstalk
1121  doBrighterFatter
1122  doDark
1123  doFringe
1124  doStrayLight
1125  doFlat
1126 
1127  In addition, setting the "postISRCCD" entry displays the
1128  exposure after all ISR processing has finished.
1129 
1130  """
1131 
1132  if isGen3 is True:
1133  # Gen3 currently cannot automatically do configuration overrides.
1134  # DM-15257 looks to discuss this issue.
1135 
1136  self.config.doFringe = False
1137 
1138  # Configure input exposures;
1139  if detectorNum is None:
1140  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1141 
1142  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1143  bias = self.ensureExposure(bias, camera, detectorNum)
1144  dark = self.ensureExposure(dark, camera, detectorNum)
1145  flat = self.ensureExposure(flat, camera, detectorNum)
1146  else:
1147  if isinstance(ccdExposure, ButlerDataRef):
1148  return self.runDataRef(ccdExposure)
1149 
1150  ccd = ccdExposure.getDetector()
1151  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1152 
1153  if not ccd:
1154  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1155  ccd = [FakeAmp(ccdExposure, self.config)]
1156 
1157  # Validate Input
1158  if self.config.doBias and bias is None:
1159  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1160  if self.doLinearize(ccd) and linearizer is None:
1161  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1162  if self.config.doBrighterFatter and bfKernel is None:
1163  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1164  if self.config.doDark and dark is None:
1165  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1166  if self.config.doFlat and flat is None:
1167  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1168  if self.config.doDefect and defects is None:
1169  raise RuntimeError("Must supply defects if config.doDefect=True.")
1170  if self.config.doAddDistortionModel and camera is None:
1171  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1172  if (self.config.doFringe and filterName in self.fringe.config.filters and
1173  fringes.fringes is None):
1174  # The `fringes` object needs to be a pipeBase.Struct, as
1175  # we use it as a `dict` for the parameters of
1176  # `FringeTask.run()`. The `fringes.fringes` `list` may
1177  # not be `None` if `doFringe=True`. Otherwise, raise.
1178  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1179  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1180  illumMaskedImage is None):
1181  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1182 
1183  # Begin ISR processing.
1184  if self.config.doConvertIntToFloat:
1185  self.log.info("Converting exposure to floating point values.")
1186  ccdExposure = self.convertIntToFloat(ccdExposure)
1187 
1188  # Amplifier level processing.
1189  overscans = []
1190  for amp in ccd:
1191  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1192  if ccdExposure.getBBox().contains(amp.getBBox()):
1193  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1194  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1195 
1196  if self.config.doOverscan and not badAmp:
1197  # Overscan correction on amp-by-amp basis.
1198  overscanResults = self.overscanCorrection(ccdExposure, amp)
1199  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1200  if overscanResults is not None and \
1201  self.config.qa is not None and self.config.qa.saveStats is True:
1202  if isinstance(overscanResults.overscanFit, float):
1203  qaMedian = overscanResults.overscanFit
1204  qaStdev = float("NaN")
1205  else:
1206  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1207  afwMath.MEDIAN | afwMath.STDEVCLIP)
1208  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1209  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1210 
1211  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1212  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1213  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1214  amp.getName(), qaMedian, qaStdev)
1215  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1216  else:
1217  if badAmp:
1218  self.log.warn("Amplifier %s is bad.", amp.getName())
1219  overscanResults = None
1220 
1221  overscans.append(overscanResults if overscanResults is not None else None)
1222  else:
1223  self.log.info("Skipped OSCAN for %s.", amp.getName())
1224 
1225  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1226  self.log.info("Applying crosstalk correction.")
1227  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1228  self.debugView(ccdExposure, "doCrosstalk")
1229 
1230  if self.config.doAssembleCcd:
1231  self.log.info("Assembling CCD from amplifiers.")
1232  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1233 
1234  if self.config.expectWcs and not ccdExposure.getWcs():
1235  self.log.warn("No WCS found in input exposure.")
1236  self.debugView(ccdExposure, "doAssembleCcd")
1237 
1238  ossThumb = None
1239  if self.config.qa.doThumbnailOss:
1240  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1241 
1242  if self.config.doBias:
1243  self.log.info("Applying bias correction.")
1244  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1245  trimToFit=self.config.doTrimToMatchCalib)
1246  self.debugView(ccdExposure, "doBias")
1247 
1248  if self.config.doVariance:
1249  for amp, overscanResults in zip(ccd, overscans):
1250  if ccdExposure.getBBox().contains(amp.getBBox()):
1251  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1252  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1253  if overscanResults is not None:
1254  self.updateVariance(ampExposure, amp,
1255  overscanImage=overscanResults.overscanImage)
1256  else:
1257  self.updateVariance(ampExposure, amp,
1258  overscanImage=None)
1259  if self.config.qa is not None and self.config.qa.saveStats is True:
1260  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1261  afwMath.MEDIAN | afwMath.STDEVCLIP)
1262  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1263  qaStats.getValue(afwMath.MEDIAN))
1264  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1265  qaStats.getValue(afwMath.STDEVCLIP))
1266  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1267  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1268  qaStats.getValue(afwMath.STDEVCLIP))
1269 
1270  if self.doLinearize(ccd):
1271  self.log.info("Applying linearizer.")
1272  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1273 
1274  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1275  self.log.info("Applying crosstalk correction.")
1276  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1277  self.debugView(ccdExposure, "doCrosstalk")
1278 
1279  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1280  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1281  if self.config.doDefect:
1282  self.log.info("Masking defects.")
1283  self.maskDefect(ccdExposure, defects)
1284 
1285  if self.config.doNanMasking:
1286  self.log.info("Masking NAN value pixels.")
1287  self.maskNan(ccdExposure)
1288 
1289  if self.config.doWidenSaturationTrails:
1290  self.log.info("Widening saturation trails.")
1291  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1292 
1293  if self.config.doCameraSpecificMasking:
1294  self.log.info("Masking regions for camera specific reasons.")
1295  self.masking.run(ccdExposure)
1296 
1297  if self.config.doBrighterFatter:
1298  # We need to apply flats and darks before we can interpolate, and we
1299  # need to interpolate before we do B-F, but we do B-F without the
1300  # flats and darks applied so we can work in units of electrons or holes.
1301  # This context manager applies and then removes the darks and flats.
1302  #
1303  # We also do not want to interpolate values here, so operate on temporary
1304  # images so we can apply only the BF-correction and roll back the
1305  # interpolation.
1306  interpExp = ccdExposure.clone()
1307  with self.flatContext(interpExp, flat, dark):
1308  isrFunctions.interpolateFromMask(
1309  maskedImage=interpExp.getMaskedImage(),
1310  fwhm=self.config.fwhm,
1311  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1312  maskNameList=self.config.maskListToInterpolate
1313  )
1314  bfExp = interpExp.clone()
1315 
1316  self.log.info("Applying brighter fatter correction.")
1317  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1318  self.config.brighterFatterMaxIter,
1319  self.config.brighterFatterThreshold,
1320  self.config.brighterFatterApplyGain
1321  )
1322  if bfResults[1] == self.config.brighterFatterMaxIter:
1323  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1324  bfResults[0])
1325  else:
1326  self.log.info("Finished brighter fatter correction in %d iterations.",
1327  bfResults[1])
1328  image = ccdExposure.getMaskedImage().getImage()
1329  bfCorr = bfExp.getMaskedImage().getImage()
1330  bfCorr -= interpExp.getMaskedImage().getImage()
1331  image += bfCorr
1332 
1333  self.debugView(ccdExposure, "doBrighterFatter")
1334 
1335  if self.config.doDark:
1336  self.log.info("Applying dark correction.")
1337  self.darkCorrection(ccdExposure, dark)
1338  self.debugView(ccdExposure, "doDark")
1339 
1340  if self.config.doFringe and not self.config.fringeAfterFlat:
1341  self.log.info("Applying fringe correction before flat.")
1342  self.fringe.run(ccdExposure, **fringes.getDict())
1343  self.debugView(ccdExposure, "doFringe")
1344 
1345  if self.config.doStrayLight:
1346  if strayLightData is not None:
1347  self.log.info("Applying stray light correction.")
1348  self.strayLight.run(ccdExposure, strayLightData)
1349  self.debugView(ccdExposure, "doStrayLight")
1350  else:
1351  self.log.debug("Skipping stray light correction: no data found for this image.")
1352 
1353  if self.config.doFlat:
1354  self.log.info("Applying flat correction.")
1355  self.flatCorrection(ccdExposure, flat)
1356  self.debugView(ccdExposure, "doFlat")
1357 
1358  if self.config.doApplyGains:
1359  self.log.info("Applying gain correction instead of flat.")
1360  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1361 
1362  if self.config.doFringe and self.config.fringeAfterFlat:
1363  self.log.info("Applying fringe correction after flat.")
1364  self.fringe.run(ccdExposure, **fringes.getDict())
1365 
1366  if self.config.doVignette:
1367  self.log.info("Constructing Vignette polygon.")
1368  self.vignettePolygon = self.vignette.run(ccdExposure)
1369 
1370  if self.config.vignette.doWriteVignettePolygon:
1371  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1372 
1373  if self.config.doAttachTransmissionCurve:
1374  self.log.info("Adding transmission curves.")
1375  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1376  filterTransmission=filterTransmission,
1377  sensorTransmission=sensorTransmission,
1378  atmosphereTransmission=atmosphereTransmission)
1379 
1380  if self.config.doAddDistortionModel:
1381  self.log.info("Adding a distortion model to the WCS.")
1382  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1383 
1384  flattenedThumb = None
1385  if self.config.qa.doThumbnailFlattened:
1386  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1387 
1388  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1389  self.log.info("Performing illumination correction.")
1390  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1391  illumMaskedImage, illumScale=self.config.illumScale,
1392  trimToFit=self.config.doTrimToMatchCalib)
1393 
1394  preInterpExp = None
1395  if self.config.doSaveInterpPixels:
1396  preInterpExp = ccdExposure.clone()
1397 
1398  # Reset and interpolate bad pixels.
1399  #
1400  # Large contiguous bad regions (which should have the BAD mask
1401  # bit set) should have their values set to the image median.
1402  # This group should include defects and bad amplifiers. As the
1403  # area covered by these defects are large, there's little
1404  # reason to expect that interpolation would provide a more
1405  # useful value.
1406  #
1407  # Smaller defects can be safely interpolated after the larger
1408  # regions have had their pixel values reset. This ensures
1409  # that the remaining defects adjacent to bad amplifiers (as an
1410  # example) do not attempt to interpolate extreme values.
1411  if self.config.doSetBadRegions:
1412  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1413  if badPixelCount > 0:
1414  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1415 
1416  if self.config.doInterpolate:
1417  self.log.info("Interpolating masked pixels.")
1418  isrFunctions.interpolateFromMask(
1419  maskedImage=ccdExposure.getMaskedImage(),
1420  fwhm=self.config.fwhm,
1421  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1422  maskNameList=list(self.config.maskListToInterpolate)
1423  )
1424 
1425  self.roughZeroPoint(ccdExposure)
1426 
1427  if self.config.doMeasureBackground:
1428  self.log.info("Measuring background level.")
1429  self.measureBackground(ccdExposure, self.config.qa)
1430 
1431  if self.config.qa is not None and self.config.qa.saveStats is True:
1432  for amp in ccd:
1433  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1434  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1435  afwMath.MEDIAN | afwMath.STDEVCLIP)
1436  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1437  qaStats.getValue(afwMath.MEDIAN))
1438  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1439  qaStats.getValue(afwMath.STDEVCLIP))
1440  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1441  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1442  qaStats.getValue(afwMath.STDEVCLIP))
1443 
1444  self.debugView(ccdExposure, "postISRCCD")
1445 
1446  return pipeBase.Struct(
1447  exposure=ccdExposure,
1448  ossThumb=ossThumb,
1449  flattenedThumb=flattenedThumb,
1450 
1451  preInterpolatedExposure=preInterpExp,
1452  outputExposure=ccdExposure,
1453  outputOssThumbnail=ossThumb,
1454  outputFlattenedThumbnail=flattenedThumb,
1455  )
1456 
1457  @pipeBase.timeMethod
1458  def runDataRef(self, sensorRef):
1459  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1460 
1461  This method contains the `CmdLineTask` interface to the ISR
1462  processing. All IO is handled here, freeing the `run()` method
1463  to manage only pixel-level calculations. The steps performed
1464  are:
1465  - Read in necessary detrending/isr/calibration data.
1466  - Process raw exposure in `run()`.
1467  - Persist the ISR-corrected exposure as "postISRCCD" if
1468  config.doWrite=True.
1469 
1470  Parameters
1471  ----------
1472  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1473  DataRef of the detector data to be processed
1474 
1475  Returns
1476  -------
1477  result : `lsst.pipe.base.Struct`
1478  Result struct with component:
1479  - ``exposure`` : `afw.image.Exposure`
1480  The fully ISR corrected exposure.
1481 
1482  Raises
1483  ------
1484  RuntimeError
1485  Raised if a configuration option is set to True, but the
1486  required calibration data does not exist.
1487 
1488  """
1489  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1490 
1491  ccdExposure = sensorRef.get(self.config.datasetType)
1492 
1493  camera = sensorRef.get("camera")
1494  if camera is None and self.config.doAddDistortionModel:
1495  raise RuntimeError("config.doAddDistortionModel is True "
1496  "but could not get a camera from the butler.")
1497  isrData = self.readIsrData(sensorRef, ccdExposure)
1498 
1499  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1500 
1501  if self.config.doWrite:
1502  sensorRef.put(result.exposure, "postISRCCD")
1503  if result.preInterpolatedExposure is not None:
1504  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1505  if result.ossThumb is not None:
1506  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1507  if result.flattenedThumb is not None:
1508  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1509 
1510  return result
1511 
1512  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1513  """!Retrieve a calibration dataset for removing instrument signature.
1514 
1515  Parameters
1516  ----------
1517 
1518  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1519  DataRef of the detector data to find calibration datasets
1520  for.
1521  datasetType : `str`
1522  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1523  immediate : `Bool`
1524  If True, disable butler proxies to enable error handling
1525  within this routine.
1526 
1527  Returns
1528  -------
1529  exposure : `lsst.afw.image.Exposure`
1530  Requested calibration frame.
1531 
1532  Raises
1533  ------
1534  RuntimeError
1535  Raised if no matching calibration frame can be found.
1536  """
1537  try:
1538  exp = dataRef.get(datasetType, immediate=immediate)
1539  except Exception as exc1:
1540  if not self.config.fallbackFilterName:
1541  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1542  try:
1543  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1544  except Exception as exc2:
1545  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1546  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1547  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1548 
1549  if self.config.doAssembleIsrExposures:
1550  exp = self.assembleCcd.assembleCcd(exp)
1551  return exp
1552 
1553  def ensureExposure(self, inputExp, camera, detectorNum):
1554  """Ensure that the data returned by Butler is a fully constructed exposure.
1555 
1556  ISR requires exposure-level image data for historical reasons, so if we did
1557  not recieve that from Butler, construct it from what we have, modifying the
1558  input in place.
1559 
1560  Parameters
1561  ----------
1562  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1563  `lsst.afw.image.ImageF`
1564  The input data structure obtained from Butler.
1565  camera : `lsst.afw.cameraGeom.camera`
1566  The camera associated with the image. Used to find the appropriate
1567  detector.
1568  detectorNum : `int`
1569  The detector this exposure should match.
1570 
1571  Returns
1572  -------
1573  inputExp : `lsst.afw.image.Exposure`
1574  The re-constructed exposure, with appropriate detector parameters.
1575 
1576  Raises
1577  ------
1578  TypeError
1579  Raised if the input data cannot be used to construct an exposure.
1580  """
1581  if isinstance(inputExp, afwImage.DecoratedImageU):
1582  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1583  elif isinstance(inputExp, afwImage.ImageF):
1584  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1585  elif isinstance(inputExp, afwImage.MaskedImageF):
1586  inputExp = afwImage.makeExposure(inputExp)
1587  elif isinstance(inputExp, afwImage.Exposure):
1588  pass
1589  elif inputExp is None:
1590  # Assume this will be caught by the setup if it is a problem.
1591  return inputExp
1592  else:
1593  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1594  (type(inputExp), ))
1595 
1596  if inputExp.getDetector() is None:
1597  inputExp.setDetector(camera[detectorNum])
1598 
1599  return inputExp
1600 
1601  def convertIntToFloat(self, exposure):
1602  """Convert exposure image from uint16 to float.
1603 
1604  If the exposure does not need to be converted, the input is
1605  immediately returned. For exposures that are converted to use
1606  floating point pixels, the variance is set to unity and the
1607  mask to zero.
1608 
1609  Parameters
1610  ----------
1611  exposure : `lsst.afw.image.Exposure`
1612  The raw exposure to be converted.
1613 
1614  Returns
1615  -------
1616  newexposure : `lsst.afw.image.Exposure`
1617  The input ``exposure``, converted to floating point pixels.
1618 
1619  Raises
1620  ------
1621  RuntimeError
1622  Raised if the exposure type cannot be converted to float.
1623 
1624  """
1625  if isinstance(exposure, afwImage.ExposureF):
1626  # Nothing to be done
1627  self.log.debug("Exposure already of type float.")
1628  return exposure
1629  if not hasattr(exposure, "convertF"):
1630  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1631 
1632  newexposure = exposure.convertF()
1633  newexposure.variance[:] = 1
1634  newexposure.mask[:] = 0x0
1635 
1636  return newexposure
1637 
1638  def maskAmplifier(self, ccdExposure, amp, defects):
1639  """Identify bad amplifiers, saturated and suspect pixels.
1640 
1641  Parameters
1642  ----------
1643  ccdExposure : `lsst.afw.image.Exposure`
1644  Input exposure to be masked.
1645  amp : `lsst.afw.table.AmpInfoCatalog`
1646  Catalog of parameters defining the amplifier on this
1647  exposure to mask.
1648  defects : `lsst.meas.algorithms.Defects`
1649  List of defects. Used to determine if the entire
1650  amplifier is bad.
1651 
1652  Returns
1653  -------
1654  badAmp : `Bool`
1655  If this is true, the entire amplifier area is covered by
1656  defects and unusable.
1657 
1658  """
1659  maskedImage = ccdExposure.getMaskedImage()
1660 
1661  badAmp = False
1662 
1663  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1664  # comparison with current defects definition.
1665  if defects is not None:
1666  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1667 
1668  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1669  # association with pixels in current ccdExposure).
1670  if badAmp:
1671  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1672  afwImage.PARENT)
1673  maskView = dataView.getMask()
1674  maskView |= maskView.getPlaneBitMask("BAD")
1675  del maskView
1676  return badAmp
1677 
1678  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1679  # Saturation and suspect pixels can be masked now, though.
1680  limits = dict()
1681  if self.config.doSaturation and not badAmp:
1682  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1683  if self.config.doSuspect and not badAmp:
1684  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1685  if math.isfinite(self.config.saturation):
1686  limits.update({self.config.saturatedMaskName: self.config.saturation})
1687 
1688  for maskName, maskThreshold in limits.items():
1689  if not math.isnan(maskThreshold):
1690  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1691  isrFunctions.makeThresholdMask(
1692  maskedImage=dataView,
1693  threshold=maskThreshold,
1694  growFootprints=0,
1695  maskName=maskName
1696  )
1697 
1698  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1699  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1700  afwImage.PARENT)
1701  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1702  self.config.suspectMaskName])
1703  if numpy.all(maskView.getArray() & maskVal > 0):
1704  badAmp = True
1705  maskView |= maskView.getPlaneBitMask("BAD")
1706 
1707  return badAmp
1708 
1709  def overscanCorrection(self, ccdExposure, amp):
1710  """Apply overscan correction in place.
1711 
1712  This method does initial pixel rejection of the overscan
1713  region. The overscan can also be optionally segmented to
1714  allow for discontinuous overscan responses to be fit
1715  separately. The actual overscan subtraction is performed by
1716  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1717  which is called here after the amplifier is preprocessed.
1718 
1719  Parameters
1720  ----------
1721  ccdExposure : `lsst.afw.image.Exposure`
1722  Exposure to have overscan correction performed.
1723  amp : `lsst.afw.table.AmpInfoCatalog`
1724  The amplifier to consider while correcting the overscan.
1725 
1726  Returns
1727  -------
1728  overscanResults : `lsst.pipe.base.Struct`
1729  Result struct with components:
1730  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1731  Value or fit subtracted from the amplifier image data.
1732  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1733  Value or fit subtracted from the overscan image data.
1734  - ``overscanImage`` : `lsst.afw.image.Image`
1735  Image of the overscan region with the overscan
1736  correction applied. This quantity is used to estimate
1737  the amplifier read noise empirically.
1738 
1739  Raises
1740  ------
1741  RuntimeError
1742  Raised if the ``amp`` does not contain raw pixel information.
1743 
1744  See Also
1745  --------
1746  lsst.ip.isr.isrFunctions.overscanCorrection
1747  """
1748  if not amp.getHasRawInfo():
1749  raise RuntimeError("This method must be executed on an amp with raw information.")
1750 
1751  if amp.getRawHorizontalOverscanBBox().isEmpty():
1752  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1753  return None
1754 
1755  statControl = afwMath.StatisticsControl()
1756  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1757 
1758  # Determine the bounding boxes
1759  dataBBox = amp.getRawDataBBox()
1760  oscanBBox = amp.getRawHorizontalOverscanBBox()
1761  dx0 = 0
1762  dx1 = 0
1763 
1764  prescanBBox = amp.getRawPrescanBBox()
1765  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1766  dx0 += self.config.overscanNumLeadingColumnsToSkip
1767  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1768  else:
1769  dx0 += self.config.overscanNumTrailingColumnsToSkip
1770  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1771 
1772  # Determine if we need to work on subregions of the amplifier and overscan.
1773  imageBBoxes = []
1774  overscanBBoxes = []
1775 
1776  if ((self.config.overscanBiasJump and
1777  self.config.overscanBiasJumpLocation) and
1778  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1779  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1780  self.config.overscanBiasJumpDevices)):
1781  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1782  yLower = self.config.overscanBiasJumpLocation
1783  yUpper = dataBBox.getHeight() - yLower
1784  else:
1785  yUpper = self.config.overscanBiasJumpLocation
1786  yLower = dataBBox.getHeight() - yUpper
1787 
1788  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1789  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1790  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1791  lsst.geom.Extent2I(dx0, 0),
1792  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1793  yLower)))
1794 
1795  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1796  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1797  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1798  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1799  yUpper)))
1800  else:
1801  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1802  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1803  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1804  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1805  oscanBBox.getHeight())))
1806 
1807  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1808  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1809  ampImage = ccdExposure.maskedImage[imageBBox]
1810  overscanImage = ccdExposure.maskedImage[overscanBBox]
1811 
1812  overscanArray = overscanImage.image.array
1813  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1814  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1815  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1816 
1817  statControl = afwMath.StatisticsControl()
1818  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1819 
1820  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1821  overscanImage=overscanImage,
1822  fitType=self.config.overscanFitType,
1823  order=self.config.overscanOrder,
1824  collapseRej=self.config.overscanNumSigmaClip,
1825  statControl=statControl,
1826  overscanIsInt=self.config.overscanIsInt
1827  )
1828 
1829  # Measure average overscan levels and record them in the metadata.
1830  levelStat = afwMath.MEDIAN
1831  sigmaStat = afwMath.STDEVCLIP
1832 
1833  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1834  self.config.qa.flatness.nIter)
1835  metadata = ccdExposure.getMetadata()
1836  ampNum = amp.getName()
1837  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1838  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1839  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1840  else:
1841  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1842  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1843  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1844 
1845  return overscanResults
1846 
1847  def updateVariance(self, ampExposure, amp, overscanImage=None):
1848  """Set the variance plane using the amplifier gain and read noise
1849 
1850  The read noise is calculated from the ``overscanImage`` if the
1851  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1852  the value from the amplifier data is used.
1853 
1854  Parameters
1855  ----------
1856  ampExposure : `lsst.afw.image.Exposure`
1857  Exposure to process.
1858  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1859  Amplifier detector data.
1860  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1861  Image of overscan, required only for empirical read noise.
1862 
1863  See also
1864  --------
1865  lsst.ip.isr.isrFunctions.updateVariance
1866  """
1867  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1868  gain = amp.getGain()
1869 
1870  if math.isnan(gain):
1871  gain = 1.0
1872  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1873  elif gain <= 0:
1874  patchedGain = 1.0
1875  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1876  amp.getName(), gain, patchedGain)
1877  gain = patchedGain
1878 
1879  if self.config.doEmpiricalReadNoise and overscanImage is None:
1880  self.log.info("Overscan is none for EmpiricalReadNoise.")
1881 
1882  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1883  stats = afwMath.StatisticsControl()
1884  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1885  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1886  self.log.info("Calculated empirical read noise for amp %s: %f.",
1887  amp.getName(), readNoise)
1888  else:
1889  readNoise = amp.getReadNoise()
1890 
1891  isrFunctions.updateVariance(
1892  maskedImage=ampExposure.getMaskedImage(),
1893  gain=gain,
1894  readNoise=readNoise,
1895  )
1896 
1897  def darkCorrection(self, exposure, darkExposure, invert=False):
1898  """!Apply dark correction in place.
1899 
1900  Parameters
1901  ----------
1902  exposure : `lsst.afw.image.Exposure`
1903  Exposure to process.
1904  darkExposure : `lsst.afw.image.Exposure`
1905  Dark exposure of the same size as ``exposure``.
1906  invert : `Bool`, optional
1907  If True, re-add the dark to an already corrected image.
1908 
1909  Raises
1910  ------
1911  RuntimeError
1912  Raised if either ``exposure`` or ``darkExposure`` do not
1913  have their dark time defined.
1914 
1915  See Also
1916  --------
1917  lsst.ip.isr.isrFunctions.darkCorrection
1918  """
1919  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1920  if math.isnan(expScale):
1921  raise RuntimeError("Exposure darktime is NAN.")
1922  if darkExposure.getInfo().getVisitInfo() is not None:
1923  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1924  else:
1925  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1926  # so getDarkTime() does not exist.
1927  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1928  darkScale = 1.0
1929 
1930  if math.isnan(darkScale):
1931  raise RuntimeError("Dark calib darktime is NAN.")
1932  isrFunctions.darkCorrection(
1933  maskedImage=exposure.getMaskedImage(),
1934  darkMaskedImage=darkExposure.getMaskedImage(),
1935  expScale=expScale,
1936  darkScale=darkScale,
1937  invert=invert,
1938  trimToFit=self.config.doTrimToMatchCalib
1939  )
1940 
1941  def doLinearize(self, detector):
1942  """!Check if linearization is needed for the detector cameraGeom.
1943 
1944  Checks config.doLinearize and the linearity type of the first
1945  amplifier.
1946 
1947  Parameters
1948  ----------
1949  detector : `lsst.afw.cameraGeom.Detector`
1950  Detector to get linearity type from.
1951 
1952  Returns
1953  -------
1954  doLinearize : `Bool`
1955  If True, linearization should be performed.
1956  """
1957  return self.config.doLinearize and \
1958  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1959 
1960  def flatCorrection(self, exposure, flatExposure, invert=False):
1961  """!Apply flat correction in place.
1962 
1963  Parameters
1964  ----------
1965  exposure : `lsst.afw.image.Exposure`
1966  Exposure to process.
1967  flatExposure : `lsst.afw.image.Exposure`
1968  Flat exposure of the same size as ``exposure``.
1969  invert : `Bool`, optional
1970  If True, unflatten an already flattened image.
1971 
1972  See Also
1973  --------
1974  lsst.ip.isr.isrFunctions.flatCorrection
1975  """
1976  isrFunctions.flatCorrection(
1977  maskedImage=exposure.getMaskedImage(),
1978  flatMaskedImage=flatExposure.getMaskedImage(),
1979  scalingType=self.config.flatScalingType,
1980  userScale=self.config.flatUserScale,
1981  invert=invert,
1982  trimToFit=self.config.doTrimToMatchCalib
1983  )
1984 
1985  def saturationDetection(self, exposure, amp):
1986  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1987 
1988  Parameters
1989  ----------
1990  exposure : `lsst.afw.image.Exposure`
1991  Exposure to process. Only the amplifier DataSec is processed.
1992  amp : `lsst.afw.table.AmpInfoCatalog`
1993  Amplifier detector data.
1994 
1995  See Also
1996  --------
1997  lsst.ip.isr.isrFunctions.makeThresholdMask
1998  """
1999  if not math.isnan(amp.getSaturation()):
2000  maskedImage = exposure.getMaskedImage()
2001  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2002  isrFunctions.makeThresholdMask(
2003  maskedImage=dataView,
2004  threshold=amp.getSaturation(),
2005  growFootprints=0,
2006  maskName=self.config.saturatedMaskName,
2007  )
2008 
2009  def saturationInterpolation(self, exposure):
2010  """!Interpolate over saturated pixels, in place.
2011 
2012  This method should be called after `saturationDetection`, to
2013  ensure that the saturated pixels have been identified in the
2014  SAT mask. It should also be called after `assembleCcd`, since
2015  saturated regions may cross amplifier boundaries.
2016 
2017  Parameters
2018  ----------
2019  exposure : `lsst.afw.image.Exposure`
2020  Exposure to process.
2021 
2022  See Also
2023  --------
2024  lsst.ip.isr.isrTask.saturationDetection
2025  lsst.ip.isr.isrFunctions.interpolateFromMask
2026  """
2027  isrFunctions.interpolateFromMask(
2028  maskedImage=exposure.getMaskedImage(),
2029  fwhm=self.config.fwhm,
2030  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2031  maskNameList=list(self.config.saturatedMaskName),
2032  )
2033 
2034  def suspectDetection(self, exposure, amp):
2035  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2036 
2037  Parameters
2038  ----------
2039  exposure : `lsst.afw.image.Exposure`
2040  Exposure to process. Only the amplifier DataSec is processed.
2041  amp : `lsst.afw.table.AmpInfoCatalog`
2042  Amplifier detector data.
2043 
2044  See Also
2045  --------
2046  lsst.ip.isr.isrFunctions.makeThresholdMask
2047 
2048  Notes
2049  -----
2050  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2051  This is intended to indicate pixels that may be affected by unknown systematics;
2052  for example if non-linearity corrections above a certain level are unstable
2053  then that would be a useful value for suspectLevel. A value of `nan` indicates
2054  that no such level exists and no pixels are to be masked as suspicious.
2055  """
2056  suspectLevel = amp.getSuspectLevel()
2057  if math.isnan(suspectLevel):
2058  return
2059 
2060  maskedImage = exposure.getMaskedImage()
2061  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2062  isrFunctions.makeThresholdMask(
2063  maskedImage=dataView,
2064  threshold=suspectLevel,
2065  growFootprints=0,
2066  maskName=self.config.suspectMaskName,
2067  )
2068 
2069  def maskDefect(self, exposure, defectBaseList):
2070  """!Mask defects using mask plane "BAD", in place.
2071 
2072  Parameters
2073  ----------
2074  exposure : `lsst.afw.image.Exposure`
2075  Exposure to process.
2076  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2077  `lsst.afw.image.DefectBase`.
2078  List of defects to mask and interpolate.
2079 
2080  Notes
2081  -----
2082  Call this after CCD assembly, since defects may cross amplifier boundaries.
2083  """
2084  maskedImage = exposure.getMaskedImage()
2085  if not isinstance(defectBaseList, Defects):
2086  # Promotes DefectBase to Defect
2087  defectList = Defects(defectBaseList)
2088  else:
2089  defectList = defectBaseList
2090  defectList.maskPixels(maskedImage, maskName="BAD")
2091 
2092  if self.config.numEdgeSuspect > 0:
2093  goodBBox = maskedImage.getBBox()
2094  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2095  goodBBox.grow(-self.config.numEdgeSuspect)
2096  # Mask pixels outside goodBBox as SUSPECT
2097  SourceDetectionTask.setEdgeBits(
2098  maskedImage,
2099  goodBBox,
2100  maskedImage.getMask().getPlaneBitMask("SUSPECT")
2101  )
2102 
2103  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2104  """Mask and interpolate defects using mask plane "BAD", in place.
2105 
2106  Parameters
2107  ----------
2108  exposure : `lsst.afw.image.Exposure`
2109  Exposure to process.
2110  defectBaseList : `List` of `Defects`
2111 
2112  """
2113  self.maskDefects(exposure, defectBaseList)
2114  isrFunctions.interpolateFromMask(
2115  maskedImage=exposure.getMaskedImage(),
2116  fwhm=self.config.fwhm,
2117  growSaturatedFootprints=0,
2118  maskNameList=["BAD"],
2119  )
2120 
2121  def maskNan(self, exposure):
2122  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2123 
2124  Parameters
2125  ----------
2126  exposure : `lsst.afw.image.Exposure`
2127  Exposure to process.
2128 
2129  Notes
2130  -----
2131  We mask over all NaNs, including those that are masked with
2132  other bits (because those may or may not be interpolated over
2133  later, and we want to remove all NaNs). Despite this
2134  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2135  the historical name.
2136  """
2137  maskedImage = exposure.getMaskedImage()
2138 
2139  # Find and mask NaNs
2140  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2141  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2142  numNans = maskNans(maskedImage, maskVal)
2143  self.metadata.set("NUMNANS", numNans)
2144  if numNans > 0:
2145  self.log.warn("There were %d unmasked NaNs.", numNans)
2146 
2147  def maskAndInterpolateNan(self, exposure):
2148  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2149 
2150  Parameters
2151  ----------
2152  exposure : `lsst.afw.image.Exposure`
2153  Exposure to process.
2154 
2155  See Also
2156  --------
2157  lsst.ip.isr.isrTask.maskNan()
2158  """
2159  self.maskNan(exposure)
2160  isrFunctions.interpolateFromMask(
2161  maskedImage=exposure.getMaskedImage(),
2162  fwhm=self.config.fwhm,
2163  growSaturatedFootprints=0,
2164  maskNameList=["UNMASKEDNAN"],
2165  )
2166 
2167  def measureBackground(self, exposure, IsrQaConfig=None):
2168  """Measure the image background in subgrids, for quality control purposes.
2169 
2170  Parameters
2171  ----------
2172  exposure : `lsst.afw.image.Exposure`
2173  Exposure to process.
2174  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2175  Configuration object containing parameters on which background
2176  statistics and subgrids to use.
2177  """
2178  if IsrQaConfig is not None:
2179  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2180  IsrQaConfig.flatness.nIter)
2181  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2182  statsControl.setAndMask(maskVal)
2183  maskedImage = exposure.getMaskedImage()
2184  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2185  skyLevel = stats.getValue(afwMath.MEDIAN)
2186  skySigma = stats.getValue(afwMath.STDEVCLIP)
2187  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2188  metadata = exposure.getMetadata()
2189  metadata.set('SKYLEVEL', skyLevel)
2190  metadata.set('SKYSIGMA', skySigma)
2191 
2192  # calcluating flatlevel over the subgrids
2193  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2194  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2195  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2196  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2197  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2198  skyLevels = numpy.zeros((nX, nY))
2199 
2200  for j in range(nY):
2201  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2202  for i in range(nX):
2203  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2204 
2205  xLLC = xc - meshXHalf
2206  yLLC = yc - meshYHalf
2207  xURC = xc + meshXHalf - 1
2208  yURC = yc + meshYHalf - 1
2209 
2210  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2211  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2212 
2213  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2214 
2215  good = numpy.where(numpy.isfinite(skyLevels))
2216  skyMedian = numpy.median(skyLevels[good])
2217  flatness = (skyLevels[good] - skyMedian) / skyMedian
2218  flatness_rms = numpy.std(flatness)
2219  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2220 
2221  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2222  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2223  nX, nY, flatness_pp, flatness_rms)
2224 
2225  metadata.set('FLATNESS_PP', float(flatness_pp))
2226  metadata.set('FLATNESS_RMS', float(flatness_rms))
2227  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2228  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2229  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2230 
2231  def roughZeroPoint(self, exposure):
2232  """Set an approximate magnitude zero point for the exposure.
2233 
2234  Parameters
2235  ----------
2236  exposure : `lsst.afw.image.Exposure`
2237  Exposure to process.
2238  """
2239  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2240  if filterName in self.config.fluxMag0T1:
2241  fluxMag0 = self.config.fluxMag0T1[filterName]
2242  else:
2243  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2244  fluxMag0 = self.config.defaultFluxMag0T1
2245 
2246  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2247  if not expTime > 0: # handle NaN as well as <= 0
2248  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2249  return
2250 
2251  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2252  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2253 
2254  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2255  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2256 
2257  Parameters
2258  ----------
2259  ccdExposure : `lsst.afw.image.Exposure`
2260  Exposure to process.
2261  fpPolygon : `lsst.afw.geom.Polygon`
2262  Polygon in focal plane coordinates.
2263  """
2264  # Get ccd corners in focal plane coordinates
2265  ccd = ccdExposure.getDetector()
2266  fpCorners = ccd.getCorners(FOCAL_PLANE)
2267  ccdPolygon = Polygon(fpCorners)
2268 
2269  # Get intersection of ccd corners with fpPolygon
2270  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2271 
2272  # Transform back to pixel positions and build new polygon
2273  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2274  validPolygon = Polygon(ccdPoints)
2275  ccdExposure.getInfo().setValidPolygon(validPolygon)
2276 
2277  @contextmanager
2278  def flatContext(self, exp, flat, dark=None):
2279  """Context manager that applies and removes flats and darks,
2280  if the task is configured to apply them.
2281 
2282  Parameters
2283  ----------
2284  exp : `lsst.afw.image.Exposure`
2285  Exposure to process.
2286  flat : `lsst.afw.image.Exposure`
2287  Flat exposure the same size as ``exp``.
2288  dark : `lsst.afw.image.Exposure`, optional
2289  Dark exposure the same size as ``exp``.
2290 
2291  Yields
2292  ------
2293  exp : `lsst.afw.image.Exposure`
2294  The flat and dark corrected exposure.
2295  """
2296  if self.config.doDark and dark is not None:
2297  self.darkCorrection(exp, dark)
2298  if self.config.doFlat:
2299  self.flatCorrection(exp, flat)
2300  try:
2301  yield exp
2302  finally:
2303  if self.config.doFlat:
2304  self.flatCorrection(exp, flat, invert=True)
2305  if self.config.doDark and dark is not None:
2306  self.darkCorrection(exp, dark, invert=True)
2307 
2308  def debugView(self, exposure, stepname):
2309  """Utility function to examine ISR exposure at different stages.
2310 
2311  Parameters
2312  ----------
2313  exposure : `lsst.afw.image.Exposure`
2314  Exposure to view.
2315  stepname : `str`
2316  State of processing to view.
2317  """
2318  frame = getDebugFrame(self._display, stepname)
2319  if frame:
2320  display = getDisplay(frame)
2321  display.scale('asinh', 'zscale')
2322  display.mtv(exposure)
2323  prompt = "Press Enter to continue [c]... "
2324  while True:
2325  ans = input(prompt).lower()
2326  if ans in ("", "c",):
2327  break
2328 
2329 
2330 class FakeAmp(object):
2331  """A Detector-like object that supports returning gain and saturation level
2332 
2333  This is used when the input exposure does not have a detector.
2334 
2335  Parameters
2336  ----------
2337  exposure : `lsst.afw.image.Exposure`
2338  Exposure to generate a fake amplifier for.
2339  config : `lsst.ip.isr.isrTaskConfig`
2340  Configuration to apply to the fake amplifier.
2341  """
2342 
2343  def __init__(self, exposure, config):
2344  self._bbox = exposure.getBBox(afwImage.LOCAL)
2346  self._gain = config.gain
2347  self._readNoise = config.readNoise
2348  self._saturation = config.saturation
2349 
2350  def getBBox(self):
2351  return self._bbox
2352 
2353  def getRawBBox(self):
2354  return self._bbox
2355 
2356  def getHasRawInfo(self):
2357  return True # but see getRawHorizontalOverscanBBox()
2358 
2360  return self._RawHorizontalOverscanBBox
2361 
2362  def getGain(self):
2363  return self._gain
2364 
2365  def getReadNoise(self):
2366  return self._readNoise
2367 
2368  def getSaturation(self):
2369  return self._saturation
2370 
2371  def getSuspectLevel(self):
2372  return float("NaN")
2373 
2374 
2375 class RunIsrConfig(pexConfig.Config):
2376  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2377 
2378 
2379 class RunIsrTask(pipeBase.CmdLineTask):
2380  """Task to wrap the default IsrTask to allow it to be retargeted.
2381 
2382  The standard IsrTask can be called directly from a command line
2383  program, but doing so removes the ability of the task to be
2384  retargeted. As most cameras override some set of the IsrTask
2385  methods, this would remove those data-specific methods in the
2386  output post-ISR images. This wrapping class fixes the issue,
2387  allowing identical post-ISR images to be generated by both the
2388  processCcd and isrTask code.
2389  """
2390  ConfigClass = RunIsrConfig
2391  _DefaultName = "runIsr"
2392 
2393  def __init__(self, *args, **kwargs):
2394  super().__init__(*args, **kwargs)
2395  self.makeSubtask("isr")
2396 
2397  def runDataRef(self, dataRef):
2398  """
2399  Parameters
2400  ----------
2401  dataRef : `lsst.daf.persistence.ButlerDataRef`
2402  data reference of the detector data to be processed
2403 
2404  Returns
2405  -------
2406  result : `pipeBase.Struct`
2407  Result struct with component:
2408 
2409  - exposure : `lsst.afw.image.Exposure`
2410  Post-ISR processed exposure.
2411  """
2412  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:769
def runDataRef(self, sensorRef)
Definition: isrTask.py:1458
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2167
def debugView(self, exposure, stepname)
Definition: isrTask.py:2308
def __init__(self, kwargs)
Definition: isrTask.py:759
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1553
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:871
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:830
def runDataRef(self, dataRef)
Definition: isrTask.py:2397
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1019
def __init__(self, args, kwargs)
Definition: isrTask.py:2393
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:821
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2231
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2103
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2359
def maskNan(self, exposure)
Definition: isrTask.py:2121
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:808
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2069
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1709
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1601
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1960
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:868
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1512
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1897
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1941
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2254
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1638
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2278
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1847
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2147
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2034
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2009
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1985
def __init__(self, exposure, config)
Definition: isrTask.py:2343