lsst.ip.isr  18.0.0-6-gfd14a2f+1
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["instrument"],
149  )
150  illumMaskedImage = pipeBase.InputDatasetField(
151  doc="Input illumination correction.",
152  name="illum",
153  scalar=True,
154  storageClass="MaskedImageF",
155  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
156  )
157 
158  # output datasets
159  outputExposure = pipeBase.OutputDatasetField(
160  doc="Output ISR processed exposure.",
161  name="postISRCCD",
162  scalar=True,
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = pipeBase.OutputDatasetField(
167  doc="Output Overscan-subtracted thumbnail image.",
168  name="OssThumb",
169  scalar=True,
170  storageClass="Thumbnail",
171  dimensions=["instrument", "visit", "detector"],
172  )
173  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
174  doc="Output flat-corrected thumbnail image.",
175  name="FlattenedThumb",
176  scalar=True,
177  storageClass="TextStorage",
178  dimensions=["instrument", "visit", "detector"],
179  )
180 
181  quantum = pipeBase.QuantumConfig(
182  dimensions=["visit", "detector", "instrument"],
183  )
184 
185 
186  datasetType = pexConfig.Field(
187  dtype=str,
188  doc="Dataset type for input data; users will typically leave this alone, "
189  "but camera-specific ISR tasks will override it",
190  default="raw",
191  )
192 
193  fallbackFilterName = pexConfig.Field(
194  dtype=str,
195  doc="Fallback default filter name for calibrations.",
196  optional=True
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
202  )
203  fwhm = pexConfig.Field(
204  dtype=float,
205  doc="FWHM of PSF in arcseconds.",
206  default=1.0,
207  )
208  qa = pexConfig.ConfigField(
209  dtype=isrQa.IsrQaConfig,
210  doc="QA related configuration options.",
211  )
212 
213  # Image conversion configuration
214  doConvertIntToFloat = pexConfig.Field(
215  dtype=bool,
216  doc="Convert integer raw images to floating point values?",
217  default=True,
218  )
219 
220  # Saturated pixel handling.
221  doSaturation = pexConfig.Field(
222  dtype=bool,
223  doc="Mask saturated pixels? NB: this is totally independent of the"
224  " interpolation option - this is ONLY setting the bits in the mask."
225  " To have them interpolated make sure doSaturationInterpolation=True",
226  default=True,
227  )
228  saturatedMaskName = pexConfig.Field(
229  dtype=str,
230  doc="Name of mask plane to use in saturation detection and interpolation",
231  default="SAT",
232  )
233  saturation = pexConfig.Field(
234  dtype=float,
235  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
236  default=float("NaN"),
237  )
238  growSaturationFootprintSize = pexConfig.Field(
239  dtype=int,
240  doc="Number of pixels by which to grow the saturation footprints",
241  default=1,
242  )
243 
244  # Suspect pixel handling.
245  doSuspect = pexConfig.Field(
246  dtype=bool,
247  doc="Mask suspect pixels?",
248  default=True,
249  )
250  suspectMaskName = pexConfig.Field(
251  dtype=str,
252  doc="Name of mask plane to use for suspect pixels",
253  default="SUSPECT",
254  )
255  numEdgeSuspect = pexConfig.Field(
256  dtype=int,
257  doc="Number of edge pixels to be flagged as untrustworthy.",
258  default=0,
259  )
260 
261  # Initial masking options.
262  doSetBadRegions = pexConfig.Field(
263  dtype=bool,
264  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
265  default=True,
266  )
267  badStatistic = pexConfig.ChoiceField(
268  dtype=str,
269  doc="How to estimate the average value for BAD regions.",
270  default='MEANCLIP',
271  allowed={
272  "MEANCLIP": "Correct using the (clipped) mean of good data",
273  "MEDIAN": "Correct using the median of the good data",
274  },
275  )
276 
277  # Overscan subtraction configuration.
278  doOverscan = pexConfig.Field(
279  dtype=bool,
280  doc="Do overscan subtraction?",
281  default=True,
282  )
283  overscanFitType = pexConfig.ChoiceField(
284  dtype=str,
285  doc="The method for fitting the overscan bias level.",
286  default='MEDIAN',
287  allowed={
288  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
289  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
290  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
291  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
292  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
293  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
294  "MEAN": "Correct using the mean of the overscan region",
295  "MEANCLIP": "Correct using a clipped mean of the overscan region",
296  "MEDIAN": "Correct using the median of the overscan region",
297  },
298  )
299  overscanOrder = pexConfig.Field(
300  dtype=int,
301  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
302  "or number of spline knots if overscan fit type is a spline."),
303  default=1,
304  )
305  overscanNumSigmaClip = pexConfig.Field(
306  dtype=float,
307  doc="Rejection threshold (sigma) for collapsing overscan before fit",
308  default=3.0,
309  )
310  overscanIsInt = pexConfig.Field(
311  dtype=bool,
312  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
313  default=True,
314  )
315  overscanNumLeadingColumnsToSkip = pexConfig.Field(
316  dtype=int,
317  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
318  default=0,
319  )
320  overscanNumTrailingColumnsToSkip = pexConfig.Field(
321  dtype=int,
322  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
323  default=0,
324  )
325  overscanMaxDev = pexConfig.Field(
326  dtype=float,
327  doc="Maximum deviation from the median for overscan",
328  default=1000.0, check=lambda x: x > 0
329  )
330  overscanBiasJump = pexConfig.Field(
331  dtype=bool,
332  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
333  default=False,
334  )
335  overscanBiasJumpKeyword = pexConfig.Field(
336  dtype=str,
337  doc="Header keyword containing information about devices.",
338  default="NO_SUCH_KEY",
339  )
340  overscanBiasJumpDevices = pexConfig.ListField(
341  dtype=str,
342  doc="List of devices that need piecewise overscan correction.",
343  default=(),
344  )
345  overscanBiasJumpLocation = pexConfig.Field(
346  dtype=int,
347  doc="Location of bias jump along y-axis.",
348  default=0,
349  )
350 
351  # Amplifier to CCD assembly configuration
352  doAssembleCcd = pexConfig.Field(
353  dtype=bool,
354  default=True,
355  doc="Assemble amp-level exposures into a ccd-level exposure?"
356  )
357  assembleCcd = pexConfig.ConfigurableField(
358  target=AssembleCcdTask,
359  doc="CCD assembly task",
360  )
361 
362  # General calibration configuration.
363  doAssembleIsrExposures = pexConfig.Field(
364  dtype=bool,
365  default=False,
366  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
367  )
368  doTrimToMatchCalib = pexConfig.Field(
369  dtype=bool,
370  default=False,
371  doc="Trim raw data to match calibration bounding boxes?"
372  )
373 
374  # Bias subtraction.
375  doBias = pexConfig.Field(
376  dtype=bool,
377  doc="Apply bias frame correction?",
378  default=True,
379  )
380  biasDataProductName = pexConfig.Field(
381  dtype=str,
382  doc="Name of the bias data product",
383  default="bias",
384  )
385 
386  # Variance construction
387  doVariance = pexConfig.Field(
388  dtype=bool,
389  doc="Calculate variance?",
390  default=True
391  )
392  gain = pexConfig.Field(
393  dtype=float,
394  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
395  default=float("NaN"),
396  )
397  readNoise = pexConfig.Field(
398  dtype=float,
399  doc="The read noise to use if no Detector is present in the Exposure",
400  default=0.0,
401  )
402  doEmpiricalReadNoise = pexConfig.Field(
403  dtype=bool,
404  default=False,
405  doc="Calculate empirical read noise instead of value from AmpInfo data?"
406  )
407 
408  # Linearization.
409  doLinearize = pexConfig.Field(
410  dtype=bool,
411  doc="Correct for nonlinearity of the detector's response?",
412  default=True,
413  )
414 
415  # Crosstalk.
416  doCrosstalk = pexConfig.Field(
417  dtype=bool,
418  doc="Apply intra-CCD crosstalk correction?",
419  default=False,
420  )
421  doCrosstalkBeforeAssemble = pexConfig.Field(
422  dtype=bool,
423  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
424  default=True,
425  )
426  crosstalk = pexConfig.ConfigurableField(
427  target=CrosstalkTask,
428  doc="Intra-CCD crosstalk correction",
429  )
430 
431  # Masking options.
432  doDefect = pexConfig.Field(
433  dtype=bool,
434  doc="Apply correction for CCD defects, e.g. hot pixels?",
435  default=True,
436  )
437  numEdgeSuspect = pexConfig.Field(
438  dtype=int,
439  doc="Number of edge pixels to be flagged as untrustworthy.",
440  default=0,
441  )
442  doNanMasking = pexConfig.Field(
443  dtype=bool,
444  doc="Mask NAN pixels?",
445  default=True,
446  )
447  doWidenSaturationTrails = pexConfig.Field(
448  dtype=bool,
449  doc="Widen bleed trails based on their width?",
450  default=True
451  )
452 
453  # Brighter-Fatter correction.
454  doBrighterFatter = pexConfig.Field(
455  dtype=bool,
456  default=False,
457  doc="Apply the brighter fatter correction"
458  )
459  brighterFatterLevel = pexConfig.ChoiceField(
460  dtype=str,
461  default="DETECTOR",
462  doc="The level at which to correct for brighter-fatter.",
463  allowed={
464  "AMP": "Every amplifier treated separately.",
465  "DETECTOR": "One kernel per detector",
466  }
467  )
468  brighterFatterKernelFile = pexConfig.Field(
469  dtype=str,
470  default='',
471  doc="Kernel file used for the brighter fatter correction"
472  )
473  brighterFatterMaxIter = pexConfig.Field(
474  dtype=int,
475  default=10,
476  doc="Maximum number of iterations for the brighter fatter correction"
477  )
478  brighterFatterThreshold = pexConfig.Field(
479  dtype=float,
480  default=1000,
481  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
482  " absolute value of the difference between the current corrected image and the one"
483  " from the previous iteration summed over all the pixels."
484  )
485  brighterFatterApplyGain = pexConfig.Field(
486  dtype=bool,
487  default=True,
488  doc="Should the gain be applied when applying the brighter fatter correction?"
489  )
490 
491  # Dark subtraction.
492  doDark = pexConfig.Field(
493  dtype=bool,
494  doc="Apply dark frame correction?",
495  default=True,
496  )
497  darkDataProductName = pexConfig.Field(
498  dtype=str,
499  doc="Name of the dark data product",
500  default="dark",
501  )
502 
503  # Camera-specific stray light removal.
504  doStrayLight = pexConfig.Field(
505  dtype=bool,
506  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
507  default=False,
508  )
509  strayLight = pexConfig.ConfigurableField(
510  target=StrayLightTask,
511  doc="y-band stray light correction"
512  )
513 
514  # Flat correction.
515  doFlat = pexConfig.Field(
516  dtype=bool,
517  doc="Apply flat field correction?",
518  default=True,
519  )
520  flatDataProductName = pexConfig.Field(
521  dtype=str,
522  doc="Name of the flat data product",
523  default="flat",
524  )
525  flatScalingType = pexConfig.ChoiceField(
526  dtype=str,
527  doc="The method for scaling the flat on the fly.",
528  default='USER',
529  allowed={
530  "USER": "Scale by flatUserScale",
531  "MEAN": "Scale by the inverse of the mean",
532  "MEDIAN": "Scale by the inverse of the median",
533  },
534  )
535  flatUserScale = pexConfig.Field(
536  dtype=float,
537  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
538  default=1.0,
539  )
540  doTweakFlat = pexConfig.Field(
541  dtype=bool,
542  doc="Tweak flats to match observed amplifier ratios?",
543  default=False
544  )
545 
546  # Amplifier normalization based on gains instead of using flats configuration.
547  doApplyGains = pexConfig.Field(
548  dtype=bool,
549  doc="Correct the amplifiers for their gains instead of applying flat correction",
550  default=False,
551  )
552  normalizeGains = pexConfig.Field(
553  dtype=bool,
554  doc="Normalize all the amplifiers in each CCD to have the same median value.",
555  default=False,
556  )
557 
558  # Fringe correction.
559  doFringe = pexConfig.Field(
560  dtype=bool,
561  doc="Apply fringe correction?",
562  default=True,
563  )
564  fringe = pexConfig.ConfigurableField(
565  target=FringeTask,
566  doc="Fringe subtraction task",
567  )
568  fringeAfterFlat = pexConfig.Field(
569  dtype=bool,
570  doc="Do fringe subtraction after flat-fielding?",
571  default=True,
572  )
573 
574  # Distortion model application.
575  doAddDistortionModel = pexConfig.Field(
576  dtype=bool,
577  doc="Apply a distortion model based on camera geometry to the WCS?",
578  default=True,
579  )
580 
581  # Initial CCD-level background statistics options.
582  doMeasureBackground = pexConfig.Field(
583  dtype=bool,
584  doc="Measure the background level on the reduced image?",
585  default=False,
586  )
587 
588  # Camera-specific masking configuration.
589  doCameraSpecificMasking = pexConfig.Field(
590  dtype=bool,
591  doc="Mask camera-specific bad regions?",
592  default=False,
593  )
594  masking = pexConfig.ConfigurableField(
595  target=MaskingTask,
596  doc="Masking task."
597  )
598 
599  # Interpolation options.
600 
601  doInterpolate = pexConfig.Field(
602  dtype=bool,
603  doc="Interpolate masked pixels?",
604  default=True,
605  )
606  doSaturationInterpolation = pexConfig.Field(
607  dtype=bool,
608  doc="Perform interpolation over pixels masked as saturated?"
609  " NB: This is independent of doSaturation; if that is False this plane"
610  " will likely be blank, resulting in a no-op here.",
611  default=True,
612  )
613  doNanInterpolation = pexConfig.Field(
614  dtype=bool,
615  doc="Perform interpolation over pixels masked as NaN?"
616  " NB: This is independent of doNanMasking; if that is False this plane"
617  " will likely be blank, resulting in a no-op here.",
618  default=True,
619  )
620  doNanInterpAfterFlat = pexConfig.Field(
621  dtype=bool,
622  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
623  "also have to interpolate them before flat-fielding."),
624  default=False,
625  )
626  maskListToInterpolate = pexConfig.ListField(
627  dtype=str,
628  doc="List of mask planes that should be interpolated.",
629  default=['SAT', 'BAD', 'UNMASKEDNAN'],
630  )
631  doSaveInterpPixels = pexConfig.Field(
632  dtype=bool,
633  doc="Save a copy of the pre-interpolated pixel values?",
634  default=False,
635  )
636 
637  # Default photometric calibration options.
638  fluxMag0T1 = pexConfig.DictField(
639  keytype=str,
640  itemtype=float,
641  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
642  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
643  ))
644  )
645  defaultFluxMag0T1 = pexConfig.Field(
646  dtype=float,
647  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
648  default=pow(10.0, 0.4*28.0)
649  )
650 
651  # Vignette correction configuration.
652  doVignette = pexConfig.Field(
653  dtype=bool,
654  doc="Apply vignetting parameters?",
655  default=False,
656  )
657  vignette = pexConfig.ConfigurableField(
658  target=VignetteTask,
659  doc="Vignetting task.",
660  )
661 
662  # Transmission curve configuration.
663  doAttachTransmissionCurve = pexConfig.Field(
664  dtype=bool,
665  default=False,
666  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
667  )
668  doUseOpticsTransmission = pexConfig.Field(
669  dtype=bool,
670  default=True,
671  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
672  )
673  doUseFilterTransmission = pexConfig.Field(
674  dtype=bool,
675  default=True,
676  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
677  )
678  doUseSensorTransmission = pexConfig.Field(
679  dtype=bool,
680  default=True,
681  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
682  )
683  doUseAtmosphereTransmission = pexConfig.Field(
684  dtype=bool,
685  default=True,
686  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
687  )
688 
689  # Illumination correction.
690  doIlluminationCorrection = pexConfig.Field(
691  dtype=bool,
692  default=False,
693  doc="Perform illumination correction?"
694  )
695  illuminationCorrectionDataProductName = pexConfig.Field(
696  dtype=str,
697  doc="Name of the illumination correction data product.",
698  default="illumcor",
699  )
700  illumScale = pexConfig.Field(
701  dtype=float,
702  doc="Scale factor for the illumination correction.",
703  default=1.0,
704  )
705  illumFilters = pexConfig.ListField(
706  dtype=str,
707  default=[],
708  doc="Only perform illumination correction for these filters."
709  )
710 
711  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
712  doWrite = pexConfig.Field(
713  dtype=bool,
714  doc="Persist postISRCCD?",
715  default=True,
716  )
717 
718  def validate(self):
719  super().validate()
720  if self.doFlat and self.doApplyGains:
721  raise ValueError("You may not specify both doFlat and doApplyGains")
722  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
723  self.config.maskListToInterpolate.append("SAT")
724  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
725  self.config.maskListToInterpolate.append("UNMASKEDNAN")
726 
727 
728 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
729  """Apply common instrument signature correction algorithms to a raw frame.
730 
731  The process for correcting imaging data is very similar from
732  camera to camera. This task provides a vanilla implementation of
733  doing these corrections, including the ability to turn certain
734  corrections off if they are not needed. The inputs to the primary
735  method, `run()`, are a raw exposure to be corrected and the
736  calibration data products. The raw input is a single chip sized
737  mosaic of all amps including overscans and other non-science
738  pixels. The method `runDataRef()` identifies and defines the
739  calibration data products, and is intended for use by a
740  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
741  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
742  subclassed for different camera, although the most camera specific
743  methods have been split into subtasks that can be redirected
744  appropriately.
745 
746  The __init__ method sets up the subtasks for ISR processing, using
747  the defaults from `lsst.ip.isr`.
748 
749  Parameters
750  ----------
751  args : `list`
752  Positional arguments passed to the Task constructor. None used at this time.
753  kwargs : `dict`, optional
754  Keyword arguments passed on to the Task constructor. None used at this time.
755  """
756  ConfigClass = IsrTaskConfig
757  _DefaultName = "isr"
758 
759  def __init__(self, **kwargs):
760  super().__init__(**kwargs)
761  self.makeSubtask("assembleCcd")
762  self.makeSubtask("crosstalk")
763  self.makeSubtask("strayLight")
764  self.makeSubtask("fringe")
765  self.makeSubtask("masking")
766  self.makeSubtask("vignette")
767 
768  @classmethod
769  def getInputDatasetTypes(cls, config):
770  inputTypeDict = super().getInputDatasetTypes(config)
771 
772  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
773  # need because the configuration tells us we will not be bothering with the
774  # correction that uses that IDT.
775  if config.doBias is not True:
776  inputTypeDict.pop("bias", None)
777  if config.doLinearize is not True:
778  inputTypeDict.pop("linearizer", None)
779  if config.doCrosstalk is not True:
780  inputTypeDict.pop("crosstalkSources", None)
781  if config.doBrighterFatter is not True:
782  inputTypeDict.pop("bfKernel", None)
783  if config.doDefect is not True:
784  inputTypeDict.pop("defects", None)
785  if config.doDark is not True:
786  inputTypeDict.pop("dark", None)
787  if config.doFlat is not True:
788  inputTypeDict.pop("flat", None)
789  if config.doAttachTransmissionCurve is not True:
790  inputTypeDict.pop("opticsTransmission", None)
791  inputTypeDict.pop("filterTransmission", None)
792  inputTypeDict.pop("sensorTransmission", None)
793  inputTypeDict.pop("atmosphereTransmission", None)
794  if config.doUseOpticsTransmission is not True:
795  inputTypeDict.pop("opticsTransmission", None)
796  if config.doUseFilterTransmission is not True:
797  inputTypeDict.pop("filterTransmission", None)
798  if config.doUseSensorTransmission is not True:
799  inputTypeDict.pop("sensorTransmission", None)
800  if config.doUseAtmosphereTransmission is not True:
801  inputTypeDict.pop("atmosphereTransmission", None)
802  if config.doIlluminationCorrection is not True:
803  inputTypeDict.pop("illumMaskedImage", None)
804 
805  return inputTypeDict
806 
807  @classmethod
808  def getOutputDatasetTypes(cls, config):
809  outputTypeDict = super().getOutputDatasetTypes(config)
810 
811  if config.qa.doThumbnailOss is not True:
812  outputTypeDict.pop("outputOssThumbnail", None)
813  if config.qa.doThumbnailFlattened is not True:
814  outputTypeDict.pop("outputFlattenedThumbnail", None)
815  if config.doWrite is not True:
816  outputTypeDict.pop("outputExposure", None)
817 
818  return outputTypeDict
819 
820  @classmethod
821  def getPrerequisiteDatasetTypes(cls, config):
822  # Input calibration datasets should not constrain the QuantumGraph
823  # (it'd be confusing if not having flats just silently resulted in no
824  # data being processed). Our nomenclature for that is that these are
825  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
826  names = set(cls.getInputDatasetTypes(config))
827  names.remove("ccdExposure")
828  return names
829 
830  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
831  try:
832  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
833  except Exception as e:
834  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
835 
836  inputData['isGen3'] = True
837 
838  if self.config.doLinearize is True:
839  if 'linearizer' not in inputData.keys():
840  detector = inputData['camera'][inputData['detectorNum']]
841  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
842  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
843 
844  if inputData['defects'] is not None:
845  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
846  # masking expects a list of defects defined by their bounding box
847  if not isinstance(inputData["defects"], Defects):
848  inputData["defects"] = Defects.fromTable(inputData["defects"])
849 
850  # Broken: DM-17169
851  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
852  # fixed for non-HSC cameras in the future.
853  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
854  # if self.config.doCrosstalk else None)
855 
856  # Broken: DM-17152
857  # Fringes are not tested to be handled correctly by Gen3 butler.
858  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
859  # assembler=self.assembleCcd
860  # if self.config.doAssembleIsrExposures else None)
861  # if self.config.doFringe and
862  # self.fringe.checkFilter(inputData['ccdExposure'])
863  # else pipeBase.Struct(fringes=None))
864 
865  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
866 
867  def makeDatasetType(self, dsConfig):
868  return super().makeDatasetType(dsConfig)
869 
870  def readIsrData(self, dataRef, rawExposure):
871  """!Retrieve necessary frames for instrument signature removal.
872 
873  Pre-fetching all required ISR data products limits the IO
874  required by the ISR. Any conflict between the calibration data
875  available and that needed for ISR is also detected prior to
876  doing processing, allowing it to fail quickly.
877 
878  Parameters
879  ----------
880  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
881  Butler reference of the detector data to be processed
882  rawExposure : `afw.image.Exposure`
883  The raw exposure that will later be corrected with the
884  retrieved calibration data; should not be modified in this
885  method.
886 
887  Returns
888  -------
889  result : `lsst.pipe.base.Struct`
890  Result struct with components (which may be `None`):
891  - ``bias``: bias calibration frame (`afw.image.Exposure`)
892  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
893  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
894  - ``dark``: dark calibration frame (`afw.image.Exposure`)
895  - ``flat``: flat calibration frame (`afw.image.Exposure`)
896  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
897  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
898  - ``fringes``: `lsst.pipe.base.Struct` with components:
899  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
900  - ``seed``: random seed derived from the ccdExposureId for random
901  number generator (`uint32`).
902  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
903  A ``TransmissionCurve`` that represents the throughput of the optics,
904  to be evaluated in focal-plane coordinates.
905  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
906  A ``TransmissionCurve`` that represents the throughput of the filter
907  itself, to be evaluated in focal-plane coordinates.
908  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
909  A ``TransmissionCurve`` that represents the throughput of the sensor
910  itself, to be evaluated in post-assembly trimmed detector coordinates.
911  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
912  A ``TransmissionCurve`` that represents the throughput of the
913  atmosphere, assumed to be spatially constant.
914  - ``strayLightData`` : `object`
915  An opaque object containing calibration information for
916  stray-light correction. If `None`, no correction will be
917  performed.
918  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
919 
920  Raises
921  ------
922  NotImplementedError :
923  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
924  """
925  ccd = rawExposure.getDetector()
926  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
927  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
928  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
929  if self.config.doBias else None)
930  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
931  linearizer = (dataRef.get("linearizer", immediate=True)
932  if self.doLinearize(ccd) else None)
933  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
934  if self.config.doCrosstalk else None)
935  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
936  if self.config.doDark else None)
937  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
938  if self.config.doFlat else None)
939 
940  brighterFatterKernel = None
941  if self.config.doBrighterFatter is True:
942 
943  # Use the new-style cp_pipe version of the kernel is it exists.
944  try:
945  brighterFatterKernel = dataRef.get("brighterFatterKernel")
946  except NoResults:
947  # Fall back to the old-style numpy-ndarray style kernel if necessary.
948  try:
949  brighterFatterKernel = dataRef.get("bfKernel")
950  except NoResults:
951  brighterFatterKernel = None
952  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
953  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
954  # this detector, or raise an error.
955  if self.config.brighterFatterLevel == 'DETECTOR':
956  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
957  else:
958  # TODO DM-15631 for implementing this
959  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
960 
961  defectList = (dataRef.get("defects")
962  if self.config.doDefect else None)
963  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
964  if self.config.doAssembleIsrExposures else None)
965  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
966  else pipeBase.Struct(fringes=None))
967 
968  if self.config.doAttachTransmissionCurve:
969  opticsTransmission = (dataRef.get("transmission_optics")
970  if self.config.doUseOpticsTransmission else None)
971  filterTransmission = (dataRef.get("transmission_filter")
972  if self.config.doUseFilterTransmission else None)
973  sensorTransmission = (dataRef.get("transmission_sensor")
974  if self.config.doUseSensorTransmission else None)
975  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
976  if self.config.doUseAtmosphereTransmission else None)
977  else:
978  opticsTransmission = None
979  filterTransmission = None
980  sensorTransmission = None
981  atmosphereTransmission = None
982 
983  if self.config.doStrayLight:
984  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
985  else:
986  strayLightData = None
987 
988  illumMaskedImage = (self.getIsrExposure(dataRef,
989  self.config.illuminationCorrectionDataProductName).getMaskedImage()
990  if (self.config.doIlluminationCorrection and
991  filterName in self.config.illumFilters)
992  else None)
993 
994  # Struct should include only kwargs to run()
995  return pipeBase.Struct(bias=biasExposure,
996  linearizer=linearizer,
997  crosstalkSources=crosstalkSources,
998  dark=darkExposure,
999  flat=flatExposure,
1000  bfKernel=brighterFatterKernel,
1001  defects=defectList,
1002  fringes=fringeStruct,
1003  opticsTransmission=opticsTransmission,
1004  filterTransmission=filterTransmission,
1005  sensorTransmission=sensorTransmission,
1006  atmosphereTransmission=atmosphereTransmission,
1007  strayLightData=strayLightData,
1008  illumMaskedImage=illumMaskedImage
1009  )
1010 
1011  @pipeBase.timeMethod
1012  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1013  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1014  opticsTransmission=None, filterTransmission=None,
1015  sensorTransmission=None, atmosphereTransmission=None,
1016  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1017  isGen3=False,
1018  ):
1019  """!Perform instrument signature removal on an exposure.
1020 
1021  Steps included in the ISR processing, in order performed, are:
1022  - saturation and suspect pixel masking
1023  - overscan subtraction
1024  - CCD assembly of individual amplifiers
1025  - bias subtraction
1026  - variance image construction
1027  - linearization of non-linear response
1028  - crosstalk masking
1029  - brighter-fatter correction
1030  - dark subtraction
1031  - fringe correction
1032  - stray light subtraction
1033  - flat correction
1034  - masking of known defects and camera specific features
1035  - vignette calculation
1036  - appending transmission curve and distortion model
1037 
1038  Parameters
1039  ----------
1040  ccdExposure : `lsst.afw.image.Exposure`
1041  The raw exposure that is to be run through ISR. The
1042  exposure is modified by this method.
1043  camera : `lsst.afw.cameraGeom.Camera`, optional
1044  The camera geometry for this exposure. Used to select the
1045  distortion model appropriate for this data.
1046  bias : `lsst.afw.image.Exposure`, optional
1047  Bias calibration frame.
1048  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1049  Functor for linearization.
1050  crosstalkSources : `list`, optional
1051  List of possible crosstalk sources.
1052  dark : `lsst.afw.image.Exposure`, optional
1053  Dark calibration frame.
1054  flat : `lsst.afw.image.Exposure`, optional
1055  Flat calibration frame.
1056  bfKernel : `numpy.ndarray`, optional
1057  Brighter-fatter kernel.
1058  defects : `lsst.meas.algorithms.Defects`, optional
1059  List of defects.
1060  fringes : `lsst.pipe.base.Struct`, optional
1061  Struct containing the fringe correction data, with
1062  elements:
1063  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1064  - ``seed``: random seed derived from the ccdExposureId for random
1065  number generator (`uint32`)
1066  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1067  A ``TransmissionCurve`` that represents the throughput of the optics,
1068  to be evaluated in focal-plane coordinates.
1069  filterTransmission : `lsst.afw.image.TransmissionCurve`
1070  A ``TransmissionCurve`` that represents the throughput of the filter
1071  itself, to be evaluated in focal-plane coordinates.
1072  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1073  A ``TransmissionCurve`` that represents the throughput of the sensor
1074  itself, to be evaluated in post-assembly trimmed detector coordinates.
1075  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1076  A ``TransmissionCurve`` that represents the throughput of the
1077  atmosphere, assumed to be spatially constant.
1078  detectorNum : `int`, optional
1079  The integer number for the detector to process.
1080  isGen3 : bool, optional
1081  Flag this call to run() as using the Gen3 butler environment.
1082  strayLightData : `object`, optional
1083  Opaque object containing calibration information for stray-light
1084  correction. If `None`, no correction will be performed.
1085  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1086  Illumination correction image.
1087 
1088  Returns
1089  -------
1090  result : `lsst.pipe.base.Struct`
1091  Result struct with component:
1092  - ``exposure`` : `afw.image.Exposure`
1093  The fully ISR corrected exposure.
1094  - ``outputExposure`` : `afw.image.Exposure`
1095  An alias for `exposure`
1096  - ``ossThumb`` : `numpy.ndarray`
1097  Thumbnail image of the exposure after overscan subtraction.
1098  - ``flattenedThumb`` : `numpy.ndarray`
1099  Thumbnail image of the exposure after flat-field correction.
1100 
1101  Raises
1102  ------
1103  RuntimeError
1104  Raised if a configuration option is set to True, but the
1105  required calibration data has not been specified.
1106 
1107  Notes
1108  -----
1109  The current processed exposure can be viewed by setting the
1110  appropriate lsstDebug entries in the `debug.display`
1111  dictionary. The names of these entries correspond to some of
1112  the IsrTaskConfig Boolean options, with the value denoting the
1113  frame to use. The exposure is shown inside the matching
1114  option check and after the processing of that step has
1115  finished. The steps with debug points are:
1116 
1117  doAssembleCcd
1118  doBias
1119  doCrosstalk
1120  doBrighterFatter
1121  doDark
1122  doFringe
1123  doStrayLight
1124  doFlat
1125 
1126  In addition, setting the "postISRCCD" entry displays the
1127  exposure after all ISR processing has finished.
1128 
1129  """
1130 
1131  if isGen3 is True:
1132  # Gen3 currently cannot automatically do configuration overrides.
1133  # DM-15257 looks to discuss this issue.
1134 
1135  self.config.doFringe = False
1136 
1137  # Configure input exposures;
1138  if detectorNum is None:
1139  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1140 
1141  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1142  bias = self.ensureExposure(bias, camera, detectorNum)
1143  dark = self.ensureExposure(dark, camera, detectorNum)
1144  flat = self.ensureExposure(flat, camera, detectorNum)
1145  else:
1146  if isinstance(ccdExposure, ButlerDataRef):
1147  return self.runDataRef(ccdExposure)
1148 
1149  ccd = ccdExposure.getDetector()
1150  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1151 
1152  if not ccd:
1153  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1154  ccd = [FakeAmp(ccdExposure, self.config)]
1155 
1156  # Validate Input
1157  if self.config.doBias and bias is None:
1158  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1159  if self.doLinearize(ccd) and linearizer is None:
1160  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1161  if self.config.doBrighterFatter and bfKernel is None:
1162  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1163  if self.config.doDark and dark is None:
1164  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1165  if self.config.doFlat and flat is None:
1166  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1167  if self.config.doDefect and defects is None:
1168  raise RuntimeError("Must supply defects if config.doDefect=True.")
1169  if self.config.doAddDistortionModel and camera is None:
1170  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1171  if (self.config.doFringe and filterName in self.fringe.config.filters and
1172  fringes.fringes is None):
1173  # The `fringes` object needs to be a pipeBase.Struct, as
1174  # we use it as a `dict` for the parameters of
1175  # `FringeTask.run()`. The `fringes.fringes` `list` may
1176  # not be `None` if `doFringe=True`. Otherwise, raise.
1177  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1178  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1179  illumMaskedImage is None):
1180  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1181 
1182  # Begin ISR processing.
1183  if self.config.doConvertIntToFloat:
1184  self.log.info("Converting exposure to floating point values.")
1185  ccdExposure = self.convertIntToFloat(ccdExposure)
1186 
1187  # Amplifier level processing.
1188  overscans = []
1189  for amp in ccd:
1190  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1191  if ccdExposure.getBBox().contains(amp.getBBox()):
1192  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1193  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1194 
1195  if self.config.doOverscan and not badAmp:
1196  # Overscan correction on amp-by-amp basis.
1197  overscanResults = self.overscanCorrection(ccdExposure, amp)
1198  self.log.debug(f"Corrected overscan for amplifier {amp.getName()}.")
1199  if overscanResults is not None and \
1200  self.config.qa is not None and self.config.qa.saveStats is True:
1201  if isinstance(overscanResults.overscanFit, float):
1202  qaMedian = overscanResults.overscanFit
1203  qaStdev = float("NaN")
1204  else:
1205  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1206  afwMath.MEDIAN | afwMath.STDEVCLIP)
1207  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1208  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1209 
1210  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1211  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1212  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1213  (amp.getName(), qaMedian, qaStdev))
1214  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1215  else:
1216  if badAmp:
1217  self.log.warn(f"Amplifier {amp.getName()} is bad.")
1218  overscanResults = None
1219 
1220  overscans.append(overscanResults if overscanResults is not None else None)
1221  else:
1222  self.log.info(f"Skipped OSCAN for {amp.getName()}.")
1223 
1224  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1225  self.log.info("Applying crosstalk correction.")
1226  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1227  self.debugView(ccdExposure, "doCrosstalk")
1228 
1229  if self.config.doAssembleCcd:
1230  self.log.info("Assembling CCD from amplifiers.")
1231  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1232 
1233  if self.config.expectWcs and not ccdExposure.getWcs():
1234  self.log.warn("No WCS found in input exposure.")
1235  self.debugView(ccdExposure, "doAssembleCcd")
1236 
1237  ossThumb = None
1238  if self.config.qa.doThumbnailOss:
1239  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1240 
1241  if self.config.doBias:
1242  self.log.info("Applying bias correction.")
1243  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1244  trimToFit=self.config.doTrimToMatchCalib)
1245  self.debugView(ccdExposure, "doBias")
1246 
1247  if self.config.doVariance:
1248  for amp, overscanResults in zip(ccd, overscans):
1249  if ccdExposure.getBBox().contains(amp.getBBox()):
1250  self.log.debug(f"Constructing variance map for amplifer {amp.getName()}.")
1251  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1252  if overscanResults is not None:
1253  self.updateVariance(ampExposure, amp,
1254  overscanImage=overscanResults.overscanImage)
1255  else:
1256  self.updateVariance(ampExposure, amp,
1257  overscanImage=None)
1258  if self.config.qa is not None and self.config.qa.saveStats is True:
1259  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1260  afwMath.MEDIAN | afwMath.STDEVCLIP)
1261  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1262  qaStats.getValue(afwMath.MEDIAN))
1263  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1264  qaStats.getValue(afwMath.STDEVCLIP))
1265  self.log.debug(" Variance stats for amplifer %s: %f +/- %f." %
1266  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1267  qaStats.getValue(afwMath.STDEVCLIP)))
1268 
1269  if self.doLinearize(ccd):
1270  self.log.info("Applying linearizer.")
1271  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1272 
1273  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1274  self.log.info("Applying crosstalk correction.")
1275  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1276  self.debugView(ccdExposure, "doCrosstalk")
1277 
1278  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1279  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1280  if self.config.doDefect:
1281  self.log.info("Masking defects.")
1282  self.maskDefect(ccdExposure, defects)
1283 
1284  if self.config.doNanMasking:
1285  self.log.info("Masking NAN value pixels.")
1286  self.maskNan(ccdExposure)
1287 
1288  if self.config.doWidenSaturationTrails:
1289  self.log.info("Widening saturation trails.")
1290  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1291 
1292  if self.config.doCameraSpecificMasking:
1293  self.log.info("Masking regions for camera specific reasons.")
1294  self.masking.run(ccdExposure)
1295 
1296  if self.config.doBrighterFatter:
1297  # We need to apply flats and darks before we can interpolate, and we
1298  # need to interpolate before we do B-F, but we do B-F without the
1299  # flats and darks applied so we can work in units of electrons or holes.
1300  # This context manager applies and then removes the darks and flats.
1301  #
1302  # We also do not want to interpolate values here, so operate on temporary
1303  # images so we can apply only the BF-correction and roll back the
1304  # interpolation.
1305  interpExp = ccdExposure.clone()
1306  with self.flatContext(interpExp, flat, dark):
1307  isrFunctions.interpolateFromMask(
1308  maskedImage=interpExp.getMaskedImage(),
1309  fwhm=self.config.fwhm,
1310  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1311  maskNameList=self.config.maskListToInterpolate
1312  )
1313  bfExp = interpExp.clone()
1314 
1315  self.log.info("Applying brighter fatter correction.")
1316  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1317  self.config.brighterFatterMaxIter,
1318  self.config.brighterFatterThreshold,
1319  self.config.brighterFatterApplyGain
1320  )
1321  if bfResults[1] == self.config.brighterFatterMaxIter:
1322  self.log.warn("Brighter fatter correction did not converge, final difference {bfResults[0]}.")
1323  else:
1324  self.log.info("Finished brighter fatter correction in {bfResults[1]} iterations.")
1325  image = ccdExposure.getMaskedImage().getImage()
1326  bfCorr = bfExp.getMaskedImage().getImage()
1327  bfCorr -= interpExp.getMaskedImage().getImage()
1328  image += bfCorr
1329 
1330  self.debugView(ccdExposure, "doBrighterFatter")
1331 
1332  if self.config.doDark:
1333  self.log.info("Applying dark correction.")
1334  self.darkCorrection(ccdExposure, dark)
1335  self.debugView(ccdExposure, "doDark")
1336 
1337  if self.config.doFringe and not self.config.fringeAfterFlat:
1338  self.log.info("Applying fringe correction before flat.")
1339  self.fringe.run(ccdExposure, **fringes.getDict())
1340  self.debugView(ccdExposure, "doFringe")
1341 
1342  if self.config.doStrayLight:
1343  if strayLightData is not None:
1344  self.log.info("Applying stray light correction.")
1345  self.strayLight.run(ccdExposure, strayLightData)
1346  self.debugView(ccdExposure, "doStrayLight")
1347  else:
1348  self.log.debug("Skipping stray light correction: no data found for this image.")
1349 
1350  if self.config.doFlat:
1351  self.log.info("Applying flat correction.")
1352  self.flatCorrection(ccdExposure, flat)
1353  self.debugView(ccdExposure, "doFlat")
1354 
1355  if self.config.doApplyGains:
1356  self.log.info("Applying gain correction instead of flat.")
1357  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1358 
1359  if self.config.doFringe and self.config.fringeAfterFlat:
1360  self.log.info("Applying fringe correction after flat.")
1361  self.fringe.run(ccdExposure, **fringes.getDict())
1362 
1363  if self.config.doVignette:
1364  self.log.info("Constructing Vignette polygon.")
1365  self.vignettePolygon = self.vignette.run(ccdExposure)
1366 
1367  if self.config.vignette.doWriteVignettePolygon:
1368  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1369 
1370  if self.config.doAttachTransmissionCurve:
1371  self.log.info("Adding transmission curves.")
1372  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1373  filterTransmission=filterTransmission,
1374  sensorTransmission=sensorTransmission,
1375  atmosphereTransmission=atmosphereTransmission)
1376 
1377  if self.config.doAddDistortionModel:
1378  self.log.info("Adding a distortion model to the WCS.")
1379  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1380 
1381  flattenedThumb = None
1382  if self.config.qa.doThumbnailFlattened:
1383  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1384 
1385  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1386  self.log.info("Performing illumination correction.")
1387  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1388  illumMaskedImage, illumScale=self.config.illumScale,
1389  trimToFit=self.config.doTrimToMatchCalib)
1390 
1391  preInterpExp = None
1392  if self.config.doSaveInterpPixels:
1393  preInterpExp = ccdExposure.clone()
1394 
1395  # Reset and interpolate bad pixels.
1396  #
1397  # Large contiguous bad regions (which should have the BAD mask
1398  # bit set) should have their values set to the image median.
1399  # This group should include defects and bad amplifiers. As the
1400  # area covered by these defects are large, there's little
1401  # reason to expect that interpolation would provide a more
1402  # useful value.
1403  #
1404  # Smaller defects can be safely interpolated after the larger
1405  # regions have had their pixel values reset. This ensures
1406  # that the remaining defects adjacent to bad amplifiers (as an
1407  # example) do not attempt to interpolate extreme values.
1408  if self.config.doSetBadRegions:
1409  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1410  if badPixelCount > 0:
1411  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1412 
1413  if self.config.doInterpolate:
1414  self.log.info("Interpolating masked pixels.")
1415  isrFunctions.interpolateFromMask(
1416  maskedImage=ccdExposure.getMaskedImage(),
1417  fwhm=self.config.fwhm,
1418  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1419  maskNameList=list(self.config.maskListToInterpolate)
1420  )
1421 
1422  self.roughZeroPoint(ccdExposure)
1423 
1424  if self.config.doMeasureBackground:
1425  self.log.info("Measuring background level.")
1426  self.measureBackground(ccdExposure, self.config.qa)
1427 
1428  if self.config.qa is not None and self.config.qa.saveStats is True:
1429  for amp in ccd:
1430  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1431  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1432  afwMath.MEDIAN | afwMath.STDEVCLIP)
1433  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1434  qaStats.getValue(afwMath.MEDIAN))
1435  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1436  qaStats.getValue(afwMath.STDEVCLIP))
1437  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1438  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1439  qaStats.getValue(afwMath.STDEVCLIP)))
1440 
1441  self.debugView(ccdExposure, "postISRCCD")
1442 
1443  return pipeBase.Struct(
1444  exposure=ccdExposure,
1445  ossThumb=ossThumb,
1446  flattenedThumb=flattenedThumb,
1447 
1448  preInterpolatedExposure=preInterpExp,
1449  outputExposure=ccdExposure,
1450  outputOssThumbnail=ossThumb,
1451  outputFlattenedThumbnail=flattenedThumb,
1452  )
1453 
1454  @pipeBase.timeMethod
1455  def runDataRef(self, sensorRef):
1456  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1457 
1458  This method contains the `CmdLineTask` interface to the ISR
1459  processing. All IO is handled here, freeing the `run()` method
1460  to manage only pixel-level calculations. The steps performed
1461  are:
1462  - Read in necessary detrending/isr/calibration data.
1463  - Process raw exposure in `run()`.
1464  - Persist the ISR-corrected exposure as "postISRCCD" if
1465  config.doWrite=True.
1466 
1467  Parameters
1468  ----------
1469  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1470  DataRef of the detector data to be processed
1471 
1472  Returns
1473  -------
1474  result : `lsst.pipe.base.Struct`
1475  Result struct with component:
1476  - ``exposure`` : `afw.image.Exposure`
1477  The fully ISR corrected exposure.
1478 
1479  Raises
1480  ------
1481  RuntimeError
1482  Raised if a configuration option is set to True, but the
1483  required calibration data does not exist.
1484 
1485  """
1486  self.log.info("Performing ISR on sensor %s." % (sensorRef.dataId))
1487 
1488  ccdExposure = sensorRef.get(self.config.datasetType)
1489 
1490  camera = sensorRef.get("camera")
1491  if camera is None and self.config.doAddDistortionModel:
1492  raise RuntimeError("config.doAddDistortionModel is True "
1493  "but could not get a camera from the butler.")
1494  isrData = self.readIsrData(sensorRef, ccdExposure)
1495 
1496  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1497 
1498  if self.config.doWrite:
1499  sensorRef.put(result.exposure, "postISRCCD")
1500  if result.preInterpolatedExposure is not None:
1501  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1502  if result.ossThumb is not None:
1503  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1504  if result.flattenedThumb is not None:
1505  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1506 
1507  return result
1508 
1509  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1510  """!Retrieve a calibration dataset for removing instrument signature.
1511 
1512  Parameters
1513  ----------
1514 
1515  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1516  DataRef of the detector data to find calibration datasets
1517  for.
1518  datasetType : `str`
1519  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1520  immediate : `Bool`
1521  If True, disable butler proxies to enable error handling
1522  within this routine.
1523 
1524  Returns
1525  -------
1526  exposure : `lsst.afw.image.Exposure`
1527  Requested calibration frame.
1528 
1529  Raises
1530  ------
1531  RuntimeError
1532  Raised if no matching calibration frame can be found.
1533  """
1534  try:
1535  exp = dataRef.get(datasetType, immediate=immediate)
1536  except Exception as exc1:
1537  if not self.config.fallbackFilterName:
1538  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1539  try:
1540  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1541  except Exception as exc2:
1542  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1543  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1544  self.log.warn("Using fallback calibration from filter %s." % self.config.fallbackFilterName)
1545 
1546  if self.config.doAssembleIsrExposures:
1547  exp = self.assembleCcd.assembleCcd(exp)
1548  return exp
1549 
1550  def ensureExposure(self, inputExp, camera, detectorNum):
1551  """Ensure that the data returned by Butler is a fully constructed exposure.
1552 
1553  ISR requires exposure-level image data for historical reasons, so if we did
1554  not recieve that from Butler, construct it from what we have, modifying the
1555  input in place.
1556 
1557  Parameters
1558  ----------
1559  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1560  `lsst.afw.image.ImageF`
1561  The input data structure obtained from Butler.
1562  camera : `lsst.afw.cameraGeom.camera`
1563  The camera associated with the image. Used to find the appropriate
1564  detector.
1565  detectorNum : `int`
1566  The detector this exposure should match.
1567 
1568  Returns
1569  -------
1570  inputExp : `lsst.afw.image.Exposure`
1571  The re-constructed exposure, with appropriate detector parameters.
1572 
1573  Raises
1574  ------
1575  TypeError
1576  Raised if the input data cannot be used to construct an exposure.
1577  """
1578  if isinstance(inputExp, afwImage.DecoratedImageU):
1579  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1580  elif isinstance(inputExp, afwImage.ImageF):
1581  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1582  elif isinstance(inputExp, afwImage.MaskedImageF):
1583  inputExp = afwImage.makeExposure(inputExp)
1584  elif isinstance(inputExp, afwImage.Exposure):
1585  pass
1586  elif inputExp is None:
1587  # Assume this will be caught by the setup if it is a problem.
1588  return inputExp
1589  else:
1590  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}.")
1591 
1592  if inputExp.getDetector() is None:
1593  inputExp.setDetector(camera[detectorNum])
1594 
1595  return inputExp
1596 
1597  def convertIntToFloat(self, exposure):
1598  """Convert exposure image from uint16 to float.
1599 
1600  If the exposure does not need to be converted, the input is
1601  immediately returned. For exposures that are converted to use
1602  floating point pixels, the variance is set to unity and the
1603  mask to zero.
1604 
1605  Parameters
1606  ----------
1607  exposure : `lsst.afw.image.Exposure`
1608  The raw exposure to be converted.
1609 
1610  Returns
1611  -------
1612  newexposure : `lsst.afw.image.Exposure`
1613  The input ``exposure``, converted to floating point pixels.
1614 
1615  Raises
1616  ------
1617  RuntimeError
1618  Raised if the exposure type cannot be converted to float.
1619 
1620  """
1621  if isinstance(exposure, afwImage.ExposureF):
1622  # Nothing to be done
1623  self.log.debug("Exposure already of type float.")
1624  return exposure
1625  if not hasattr(exposure, "convertF"):
1626  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1627 
1628  newexposure = exposure.convertF()
1629  newexposure.variance[:] = 1
1630  newexposure.mask[:] = 0x0
1631 
1632  return newexposure
1633 
1634  def maskAmplifier(self, ccdExposure, amp, defects):
1635  """Identify bad amplifiers, saturated and suspect pixels.
1636 
1637  Parameters
1638  ----------
1639  ccdExposure : `lsst.afw.image.Exposure`
1640  Input exposure to be masked.
1641  amp : `lsst.afw.table.AmpInfoCatalog`
1642  Catalog of parameters defining the amplifier on this
1643  exposure to mask.
1644  defects : `lsst.meas.algorithms.Defects`
1645  List of defects. Used to determine if the entire
1646  amplifier is bad.
1647 
1648  Returns
1649  -------
1650  badAmp : `Bool`
1651  If this is true, the entire amplifier area is covered by
1652  defects and unusable.
1653 
1654  """
1655  maskedImage = ccdExposure.getMaskedImage()
1656 
1657  badAmp = False
1658 
1659  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1660  # comparison with current defects definition.
1661  if defects is not None:
1662  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1663 
1664  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1665  # association with pixels in current ccdExposure).
1666  if badAmp:
1667  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1668  afwImage.PARENT)
1669  maskView = dataView.getMask()
1670  maskView |= maskView.getPlaneBitMask("BAD")
1671  del maskView
1672  return badAmp
1673 
1674  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1675  # Saturation and suspect pixels can be masked now, though.
1676  limits = dict()
1677  if self.config.doSaturation and not badAmp:
1678  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1679  if self.config.doSuspect and not badAmp:
1680  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1681  if math.isfinite(self.config.saturation):
1682  limits.update({self.config.saturatedMaskName: self.config.saturation})
1683 
1684  for maskName, maskThreshold in limits.items():
1685  if not math.isnan(maskThreshold):
1686  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1687  isrFunctions.makeThresholdMask(
1688  maskedImage=dataView,
1689  threshold=maskThreshold,
1690  growFootprints=0,
1691  maskName=maskName
1692  )
1693 
1694  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1695  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1696  afwImage.PARENT)
1697  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1698  self.config.suspectMaskName])
1699  if numpy.all(maskView.getArray() & maskVal > 0):
1700  badAmp = True
1701  maskView |= maskView.getPlaneBitMask("BAD")
1702 
1703  return badAmp
1704 
1705  def overscanCorrection(self, ccdExposure, amp):
1706  """Apply overscan correction in place.
1707 
1708  This method does initial pixel rejection of the overscan
1709  region. The overscan can also be optionally segmented to
1710  allow for discontinuous overscan responses to be fit
1711  separately. The actual overscan subtraction is performed by
1712  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1713  which is called here after the amplifier is preprocessed.
1714 
1715  Parameters
1716  ----------
1717  ccdExposure : `lsst.afw.image.Exposure`
1718  Exposure to have overscan correction performed.
1719  amp : `lsst.afw.table.AmpInfoCatalog`
1720  The amplifier to consider while correcting the overscan.
1721 
1722  Returns
1723  -------
1724  overscanResults : `lsst.pipe.base.Struct`
1725  Result struct with components:
1726  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1727  Value or fit subtracted from the amplifier image data.
1728  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1729  Value or fit subtracted from the overscan image data.
1730  - ``overscanImage`` : `lsst.afw.image.Image`
1731  Image of the overscan region with the overscan
1732  correction applied. This quantity is used to estimate
1733  the amplifier read noise empirically.
1734 
1735  Raises
1736  ------
1737  RuntimeError
1738  Raised if the ``amp`` does not contain raw pixel information.
1739 
1740  See Also
1741  --------
1742  lsst.ip.isr.isrFunctions.overscanCorrection
1743  """
1744  if not amp.getHasRawInfo():
1745  raise RuntimeError("This method must be executed on an amp with raw information.")
1746 
1747  if amp.getRawHorizontalOverscanBBox().isEmpty():
1748  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1749  return None
1750 
1751  statControl = afwMath.StatisticsControl()
1752  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1753 
1754  # Determine the bounding boxes
1755  dataBBox = amp.getRawDataBBox()
1756  oscanBBox = amp.getRawHorizontalOverscanBBox()
1757  dx0 = 0
1758  dx1 = 0
1759 
1760  prescanBBox = amp.getRawPrescanBBox()
1761  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1762  dx0 += self.config.overscanNumLeadingColumnsToSkip
1763  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1764  else:
1765  dx0 += self.config.overscanNumTrailingColumnsToSkip
1766  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1767 
1768  # Determine if we need to work on subregions of the amplifier and overscan.
1769  imageBBoxes = []
1770  overscanBBoxes = []
1771 
1772  if ((self.config.overscanBiasJump and
1773  self.config.overscanBiasJumpLocation) and
1774  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1775  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1776  self.config.overscanBiasJumpDevices)):
1777  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1778  yLower = self.config.overscanBiasJumpLocation
1779  yUpper = dataBBox.getHeight() - yLower
1780  else:
1781  yUpper = self.config.overscanBiasJumpLocation
1782  yLower = dataBBox.getHeight() - yUpper
1783 
1784  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1785  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1786  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1787  lsst.geom.Extent2I(dx0, 0),
1788  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1789  yLower)))
1790 
1791  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1792  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1793  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1794  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1795  yUpper)))
1796  else:
1797  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1798  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1799  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1800  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1801  oscanBBox.getHeight())))
1802 
1803  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1804  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1805  ampImage = ccdExposure.maskedImage[imageBBox]
1806  overscanImage = ccdExposure.maskedImage[overscanBBox]
1807 
1808  overscanArray = overscanImage.image.array
1809  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1810  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1811  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1812 
1813  statControl = afwMath.StatisticsControl()
1814  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1815 
1816  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1817  overscanImage=overscanImage,
1818  fitType=self.config.overscanFitType,
1819  order=self.config.overscanOrder,
1820  collapseRej=self.config.overscanNumSigmaClip,
1821  statControl=statControl,
1822  overscanIsInt=self.config.overscanIsInt
1823  )
1824 
1825  # Measure average overscan levels and record them in the metadata.
1826  levelStat = afwMath.MEDIAN
1827  sigmaStat = afwMath.STDEVCLIP
1828 
1829  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1830  self.config.qa.flatness.nIter)
1831  metadata = ccdExposure.getMetadata()
1832  ampNum = amp.getName()
1833  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1834  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1835  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1836  else:
1837  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1838  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1839  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1840 
1841  return overscanResults
1842 
1843  def updateVariance(self, ampExposure, amp, overscanImage=None):
1844  """Set the variance plane using the amplifier gain and read noise
1845 
1846  The read noise is calculated from the ``overscanImage`` if the
1847  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1848  the value from the amplifier data is used.
1849 
1850  Parameters
1851  ----------
1852  ampExposure : `lsst.afw.image.Exposure`
1853  Exposure to process.
1854  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1855  Amplifier detector data.
1856  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1857  Image of overscan, required only for empirical read noise.
1858 
1859  See also
1860  --------
1861  lsst.ip.isr.isrFunctions.updateVariance
1862  """
1863  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1864  gain = amp.getGain()
1865 
1866  if math.isnan(gain):
1867  gain = 1.0
1868  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1869  elif gain <= 0:
1870  patchedGain = 1.0
1871  self.log.warn("Gain for amp %s == %g <= 0; setting to %f." %
1872  (amp.getName(), gain, patchedGain))
1873  gain = patchedGain
1874 
1875  if self.config.doEmpiricalReadNoise and overscanImage is None:
1876  self.log.info("Overscan is none for EmpiricalReadNoise.")
1877 
1878  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1879  stats = afwMath.StatisticsControl()
1880  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1881  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1882  self.log.info("Calculated empirical read noise for amp %s: %f.", amp.getName(), readNoise)
1883  else:
1884  readNoise = amp.getReadNoise()
1885 
1886  isrFunctions.updateVariance(
1887  maskedImage=ampExposure.getMaskedImage(),
1888  gain=gain,
1889  readNoise=readNoise,
1890  )
1891 
1892  def darkCorrection(self, exposure, darkExposure, invert=False):
1893  """!Apply dark correction in place.
1894 
1895  Parameters
1896  ----------
1897  exposure : `lsst.afw.image.Exposure`
1898  Exposure to process.
1899  darkExposure : `lsst.afw.image.Exposure`
1900  Dark exposure of the same size as ``exposure``.
1901  invert : `Bool`, optional
1902  If True, re-add the dark to an already corrected image.
1903 
1904  Raises
1905  ------
1906  RuntimeError
1907  Raised if either ``exposure`` or ``darkExposure`` do not
1908  have their dark time defined.
1909 
1910  See Also
1911  --------
1912  lsst.ip.isr.isrFunctions.darkCorrection
1913  """
1914  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1915  if math.isnan(expScale):
1916  raise RuntimeError("Exposure darktime is NAN.")
1917  if darkExposure.getInfo().getVisitInfo() is not None:
1918  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1919  else:
1920  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1921  # so getDarkTime() does not exist.
1922  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1923  darkScale = 1.0
1924 
1925  if math.isnan(darkScale):
1926  raise RuntimeError("Dark calib darktime is NAN.")
1927  isrFunctions.darkCorrection(
1928  maskedImage=exposure.getMaskedImage(),
1929  darkMaskedImage=darkExposure.getMaskedImage(),
1930  expScale=expScale,
1931  darkScale=darkScale,
1932  invert=invert,
1933  trimToFit=self.config.doTrimToMatchCalib
1934  )
1935 
1936  def doLinearize(self, detector):
1937  """!Check if linearization is needed for the detector cameraGeom.
1938 
1939  Checks config.doLinearize and the linearity type of the first
1940  amplifier.
1941 
1942  Parameters
1943  ----------
1944  detector : `lsst.afw.cameraGeom.Detector`
1945  Detector to get linearity type from.
1946 
1947  Returns
1948  -------
1949  doLinearize : `Bool`
1950  If True, linearization should be performed.
1951  """
1952  return self.config.doLinearize and \
1953  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1954 
1955  def flatCorrection(self, exposure, flatExposure, invert=False):
1956  """!Apply flat correction in place.
1957 
1958  Parameters
1959  ----------
1960  exposure : `lsst.afw.image.Exposure`
1961  Exposure to process.
1962  flatExposure : `lsst.afw.image.Exposure`
1963  Flat exposure of the same size as ``exposure``.
1964  invert : `Bool`, optional
1965  If True, unflatten an already flattened image.
1966 
1967  See Also
1968  --------
1969  lsst.ip.isr.isrFunctions.flatCorrection
1970  """
1971  isrFunctions.flatCorrection(
1972  maskedImage=exposure.getMaskedImage(),
1973  flatMaskedImage=flatExposure.getMaskedImage(),
1974  scalingType=self.config.flatScalingType,
1975  userScale=self.config.flatUserScale,
1976  invert=invert,
1977  trimToFit=self.config.doTrimToMatchCalib
1978  )
1979 
1980  def saturationDetection(self, exposure, amp):
1981  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1982 
1983  Parameters
1984  ----------
1985  exposure : `lsst.afw.image.Exposure`
1986  Exposure to process. Only the amplifier DataSec is processed.
1987  amp : `lsst.afw.table.AmpInfoCatalog`
1988  Amplifier detector data.
1989 
1990  See Also
1991  --------
1992  lsst.ip.isr.isrFunctions.makeThresholdMask
1993  """
1994  if not math.isnan(amp.getSaturation()):
1995  maskedImage = exposure.getMaskedImage()
1996  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1997  isrFunctions.makeThresholdMask(
1998  maskedImage=dataView,
1999  threshold=amp.getSaturation(),
2000  growFootprints=0,
2001  maskName=self.config.saturatedMaskName,
2002  )
2003 
2004  def saturationInterpolation(self, exposure):
2005  """!Interpolate over saturated pixels, in place.
2006 
2007  This method should be called after `saturationDetection`, to
2008  ensure that the saturated pixels have been identified in the
2009  SAT mask. It should also be called after `assembleCcd`, since
2010  saturated regions may cross amplifier boundaries.
2011 
2012  Parameters
2013  ----------
2014  exposure : `lsst.afw.image.Exposure`
2015  Exposure to process.
2016 
2017  See Also
2018  --------
2019  lsst.ip.isr.isrTask.saturationDetection
2020  lsst.ip.isr.isrFunctions.interpolateFromMask
2021  """
2022  isrFunctions.interpolateFromMask(
2023  maskedImage=exposure.getMaskedImage(),
2024  fwhm=self.config.fwhm,
2025  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2026  maskNameList=list(self.config.saturatedMaskName),
2027  )
2028 
2029  def suspectDetection(self, exposure, amp):
2030  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2031 
2032  Parameters
2033  ----------
2034  exposure : `lsst.afw.image.Exposure`
2035  Exposure to process. Only the amplifier DataSec is processed.
2036  amp : `lsst.afw.table.AmpInfoCatalog`
2037  Amplifier detector data.
2038 
2039  See Also
2040  --------
2041  lsst.ip.isr.isrFunctions.makeThresholdMask
2042 
2043  Notes
2044  -----
2045  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2046  This is intended to indicate pixels that may be affected by unknown systematics;
2047  for example if non-linearity corrections above a certain level are unstable
2048  then that would be a useful value for suspectLevel. A value of `nan` indicates
2049  that no such level exists and no pixels are to be masked as suspicious.
2050  """
2051  suspectLevel = amp.getSuspectLevel()
2052  if math.isnan(suspectLevel):
2053  return
2054 
2055  maskedImage = exposure.getMaskedImage()
2056  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2057  isrFunctions.makeThresholdMask(
2058  maskedImage=dataView,
2059  threshold=suspectLevel,
2060  growFootprints=0,
2061  maskName=self.config.suspectMaskName,
2062  )
2063 
2064  def maskDefect(self, exposure, defectBaseList):
2065  """!Mask defects using mask plane "BAD", in place.
2066 
2067  Parameters
2068  ----------
2069  exposure : `lsst.afw.image.Exposure`
2070  Exposure to process.
2071  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2072  `lsst.afw.image.DefectBase`.
2073  List of defects to mask and interpolate.
2074 
2075  Notes
2076  -----
2077  Call this after CCD assembly, since defects may cross amplifier boundaries.
2078  """
2079  maskedImage = exposure.getMaskedImage()
2080  if not isinstance(defectBaseList, Defects):
2081  # Promotes DefectBase to Defect
2082  defectList = Defects(defectBaseList)
2083  else:
2084  defectList = defectBaseList
2085  defectList.maskPixels(maskedImage, maskName="BAD")
2086 
2087  if self.config.numEdgeSuspect > 0:
2088  goodBBox = maskedImage.getBBox()
2089  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2090  goodBBox.grow(-self.config.numEdgeSuspect)
2091  # Mask pixels outside goodBBox as SUSPECT
2092  SourceDetectionTask.setEdgeBits(
2093  maskedImage,
2094  goodBBox,
2095  maskedImage.getMask().getPlaneBitMask("SUSPECT")
2096  )
2097 
2098  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2099  """Mask and interpolate defects using mask plane "BAD", in place.
2100 
2101  Parameters
2102  ----------
2103  exposure : `lsst.afw.image.Exposure`
2104  Exposure to process.
2105  defectBaseList : `List` of `Defects`
2106 
2107  """
2108  self.maskDefects(exposure, defectBaseList)
2109  isrFunctions.interpolateFromMask(
2110  maskedImage=exposure.getMaskedImage(),
2111  fwhm=self.config.fwhm,
2112  growSaturatedFootprints=0,
2113  maskNameList=["BAD"],
2114  )
2115 
2116  def maskNan(self, exposure):
2117  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2118 
2119  Parameters
2120  ----------
2121  exposure : `lsst.afw.image.Exposure`
2122  Exposure to process.
2123 
2124  Notes
2125  -----
2126  We mask over all NaNs, including those that are masked with
2127  other bits (because those may or may not be interpolated over
2128  later, and we want to remove all NaNs). Despite this
2129  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2130  the historical name.
2131  """
2132  maskedImage = exposure.getMaskedImage()
2133 
2134  # Find and mask NaNs
2135  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2136  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2137  numNans = maskNans(maskedImage, maskVal)
2138  self.metadata.set("NUMNANS", numNans)
2139  if numNans > 0:
2140  self.log.warn(f"There were {numNans} unmasked NaNs.")
2141 
2142  def maskAndInterpolateNan(self, exposure):
2143  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2144 
2145  Parameters
2146  ----------
2147  exposure : `lsst.afw.image.Exposure`
2148  Exposure to process.
2149 
2150  See Also
2151  --------
2152  lsst.ip.isr.isrTask.maskNan()
2153  """
2154  self.maskNan(exposure)
2155  isrFunctions.interpolateFromMask(
2156  maskedImage=exposure.getMaskedImage(),
2157  fwhm=self.config.fwhm,
2158  growSaturatedFootprints=0,
2159  maskNameList=["UNMASKEDNAN"],
2160  )
2161 
2162  def measureBackground(self, exposure, IsrQaConfig=None):
2163  """Measure the image background in subgrids, for quality control purposes.
2164 
2165  Parameters
2166  ----------
2167  exposure : `lsst.afw.image.Exposure`
2168  Exposure to process.
2169  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2170  Configuration object containing parameters on which background
2171  statistics and subgrids to use.
2172  """
2173  if IsrQaConfig is not None:
2174  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2175  IsrQaConfig.flatness.nIter)
2176  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2177  statsControl.setAndMask(maskVal)
2178  maskedImage = exposure.getMaskedImage()
2179  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2180  skyLevel = stats.getValue(afwMath.MEDIAN)
2181  skySigma = stats.getValue(afwMath.STDEVCLIP)
2182  self.log.info("Flattened sky level: %f +/- %f." % (skyLevel, skySigma))
2183  metadata = exposure.getMetadata()
2184  metadata.set('SKYLEVEL', skyLevel)
2185  metadata.set('SKYSIGMA', skySigma)
2186 
2187  # calcluating flatlevel over the subgrids
2188  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2189  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2190  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2191  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2192  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2193  skyLevels = numpy.zeros((nX, nY))
2194 
2195  for j in range(nY):
2196  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2197  for i in range(nX):
2198  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2199 
2200  xLLC = xc - meshXHalf
2201  yLLC = yc - meshYHalf
2202  xURC = xc + meshXHalf - 1
2203  yURC = yc + meshYHalf - 1
2204 
2205  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2206  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2207 
2208  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2209 
2210  good = numpy.where(numpy.isfinite(skyLevels))
2211  skyMedian = numpy.median(skyLevels[good])
2212  flatness = (skyLevels[good] - skyMedian) / skyMedian
2213  flatness_rms = numpy.std(flatness)
2214  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2215 
2216  self.log.info("Measuring sky levels in %dx%d grids: %f." % (nX, nY, skyMedian))
2217  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f." %
2218  (nX, nY, flatness_pp, flatness_rms))
2219 
2220  metadata.set('FLATNESS_PP', float(flatness_pp))
2221  metadata.set('FLATNESS_RMS', float(flatness_rms))
2222  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2223  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2224  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2225 
2226  def roughZeroPoint(self, exposure):
2227  """Set an approximate magnitude zero point for the exposure.
2228 
2229  Parameters
2230  ----------
2231  exposure : `lsst.afw.image.Exposure`
2232  Exposure to process.
2233  """
2234  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2235  if filterName in self.config.fluxMag0T1:
2236  fluxMag0 = self.config.fluxMag0T1[filterName]
2237  else:
2238  self.log.warn("No rough magnitude zero point set for filter %s." % filterName)
2239  fluxMag0 = self.config.defaultFluxMag0T1
2240 
2241  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2242  if not expTime > 0: # handle NaN as well as <= 0
2243  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2244  return
2245 
2246  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2247  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2248 
2249  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2250  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2251 
2252  Parameters
2253  ----------
2254  ccdExposure : `lsst.afw.image.Exposure`
2255  Exposure to process.
2256  fpPolygon : `lsst.afw.geom.Polygon`
2257  Polygon in focal plane coordinates.
2258  """
2259  # Get ccd corners in focal plane coordinates
2260  ccd = ccdExposure.getDetector()
2261  fpCorners = ccd.getCorners(FOCAL_PLANE)
2262  ccdPolygon = Polygon(fpCorners)
2263 
2264  # Get intersection of ccd corners with fpPolygon
2265  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2266 
2267  # Transform back to pixel positions and build new polygon
2268  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2269  validPolygon = Polygon(ccdPoints)
2270  ccdExposure.getInfo().setValidPolygon(validPolygon)
2271 
2272  @contextmanager
2273  def flatContext(self, exp, flat, dark=None):
2274  """Context manager that applies and removes flats and darks,
2275  if the task is configured to apply them.
2276 
2277  Parameters
2278  ----------
2279  exp : `lsst.afw.image.Exposure`
2280  Exposure to process.
2281  flat : `lsst.afw.image.Exposure`
2282  Flat exposure the same size as ``exp``.
2283  dark : `lsst.afw.image.Exposure`, optional
2284  Dark exposure the same size as ``exp``.
2285 
2286  Yields
2287  ------
2288  exp : `lsst.afw.image.Exposure`
2289  The flat and dark corrected exposure.
2290  """
2291  if self.config.doDark and dark is not None:
2292  self.darkCorrection(exp, dark)
2293  if self.config.doFlat:
2294  self.flatCorrection(exp, flat)
2295  try:
2296  yield exp
2297  finally:
2298  if self.config.doFlat:
2299  self.flatCorrection(exp, flat, invert=True)
2300  if self.config.doDark and dark is not None:
2301  self.darkCorrection(exp, dark, invert=True)
2302 
2303  def debugView(self, exposure, stepname):
2304  """Utility function to examine ISR exposure at different stages.
2305 
2306  Parameters
2307  ----------
2308  exposure : `lsst.afw.image.Exposure`
2309  Exposure to view.
2310  stepname : `str`
2311  State of processing to view.
2312  """
2313  frame = getDebugFrame(self._display, stepname)
2314  if frame:
2315  display = getDisplay(frame)
2316  display.scale('asinh', 'zscale')
2317  display.mtv(exposure)
2318  prompt = "Press Enter to continue [c]... "
2319  while True:
2320  ans = input(prompt).lower()
2321  if ans in ("", "c",):
2322  break
2323 
2324 
2325 class FakeAmp(object):
2326  """A Detector-like object that supports returning gain and saturation level
2327 
2328  This is used when the input exposure does not have a detector.
2329 
2330  Parameters
2331  ----------
2332  exposure : `lsst.afw.image.Exposure`
2333  Exposure to generate a fake amplifier for.
2334  config : `lsst.ip.isr.isrTaskConfig`
2335  Configuration to apply to the fake amplifier.
2336  """
2337 
2338  def __init__(self, exposure, config):
2339  self._bbox = exposure.getBBox(afwImage.LOCAL)
2341  self._gain = config.gain
2342  self._readNoise = config.readNoise
2343  self._saturation = config.saturation
2344 
2345  def getBBox(self):
2346  return self._bbox
2347 
2348  def getRawBBox(self):
2349  return self._bbox
2350 
2351  def getHasRawInfo(self):
2352  return True # but see getRawHorizontalOverscanBBox()
2353 
2355  return self._RawHorizontalOverscanBBox
2356 
2357  def getGain(self):
2358  return self._gain
2359 
2360  def getReadNoise(self):
2361  return self._readNoise
2362 
2363  def getSaturation(self):
2364  return self._saturation
2365 
2366  def getSuspectLevel(self):
2367  return float("NaN")
2368 
2369 
2370 class RunIsrConfig(pexConfig.Config):
2371  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2372 
2373 
2374 class RunIsrTask(pipeBase.CmdLineTask):
2375  """Task to wrap the default IsrTask to allow it to be retargeted.
2376 
2377  The standard IsrTask can be called directly from a command line
2378  program, but doing so removes the ability of the task to be
2379  retargeted. As most cameras override some set of the IsrTask
2380  methods, this would remove those data-specific methods in the
2381  output post-ISR images. This wrapping class fixes the issue,
2382  allowing identical post-ISR images to be generated by both the
2383  processCcd and isrTask code.
2384  """
2385  ConfigClass = RunIsrConfig
2386  _DefaultName = "runIsr"
2387 
2388  def __init__(self, *args, **kwargs):
2389  super().__init__(*args, **kwargs)
2390  self.makeSubtask("isr")
2391 
2392  def runDataRef(self, dataRef):
2393  """
2394  Parameters
2395  ----------
2396  dataRef : `lsst.daf.persistence.ButlerDataRef`
2397  data reference of the detector data to be processed
2398 
2399  Returns
2400  -------
2401  result : `pipeBase.Struct`
2402  Result struct with component:
2403 
2404  - exposure : `lsst.afw.image.Exposure`
2405  Post-ISR processed exposure.
2406  """
2407  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:769
def runDataRef(self, sensorRef)
Definition: isrTask.py:1455
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2162
def debugView(self, exposure, stepname)
Definition: isrTask.py:2303
def __init__(self, kwargs)
Definition: isrTask.py:759
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1550
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:870
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:830
def runDataRef(self, dataRef)
Definition: isrTask.py:2392
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1018
def __init__(self, args, kwargs)
Definition: isrTask.py:2388
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:821
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2226
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2098
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2354
def maskNan(self, exposure)
Definition: isrTask.py:2116
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:808
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2064
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1705
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1597
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1955
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:867
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1509
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1892
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1936
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2249
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1634
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2273
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1843
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2142
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2029
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2004
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1980
def __init__(self, exposure, config)
Definition: isrTask.py:2338