lsst.ip.isr  17.0.1-25-g86f9585
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["instrument"],
149  )
150  illumMaskedImage = pipeBase.InputDatasetField(
151  doc="Input illumination correction.",
152  name="illum",
153  scalar=True,
154  storageClass="MaskedImageF",
155  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
156  )
157 
158  # output datasets
159  outputExposure = pipeBase.OutputDatasetField(
160  doc="Output ISR processed exposure.",
161  name="postISRCCD",
162  scalar=True,
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = pipeBase.OutputDatasetField(
167  doc="Output Overscan-subtracted thumbnail image.",
168  name="OssThumb",
169  scalar=True,
170  storageClass="Thumbnail",
171  dimensions=["instrument", "visit", "detector"],
172  )
173  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
174  doc="Output flat-corrected thumbnail image.",
175  name="FlattenedThumb",
176  scalar=True,
177  storageClass="TextStorage",
178  dimensions=["instrument", "visit", "detector"],
179  )
180 
181  quantum = pipeBase.QuantumConfig(
182  dimensions=["visit", "detector", "instrument"],
183  )
184 
185 
186  datasetType = pexConfig.Field(
187  dtype=str,
188  doc="Dataset type for input data; users will typically leave this alone, "
189  "but camera-specific ISR tasks will override it",
190  default="raw",
191  )
192 
193  fallbackFilterName = pexConfig.Field(
194  dtype=str,
195  doc="Fallback default filter name for calibrations.",
196  optional=True
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
202  )
203  fwhm = pexConfig.Field(
204  dtype=float,
205  doc="FWHM of PSF in arcseconds.",
206  default=1.0,
207  )
208  qa = pexConfig.ConfigField(
209  dtype=isrQa.IsrQaConfig,
210  doc="QA related configuration options.",
211  )
212 
213  # Image conversion configuration
214  doConvertIntToFloat = pexConfig.Field(
215  dtype=bool,
216  doc="Convert integer raw images to floating point values?",
217  default=True,
218  )
219 
220  # Saturated pixel handling.
221  doSaturation = pexConfig.Field(
222  dtype=bool,
223  doc="Mask saturated pixels? NB: this is totally independent of the"
224  " interpolation option - this is ONLY setting the bits in the mask."
225  " To have them interpolated make sure doSaturationInterpolation=True",
226  default=True,
227  )
228  saturatedMaskName = pexConfig.Field(
229  dtype=str,
230  doc="Name of mask plane to use in saturation detection and interpolation",
231  default="SAT",
232  )
233  saturation = pexConfig.Field(
234  dtype=float,
235  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
236  default=float("NaN"),
237  )
238  growSaturationFootprintSize = pexConfig.Field(
239  dtype=int,
240  doc="Number of pixels by which to grow the saturation footprints",
241  default=1,
242  )
243 
244  # Suspect pixel handling.
245  doSuspect = pexConfig.Field(
246  dtype=bool,
247  doc="Mask suspect pixels?",
248  default=True,
249  )
250  suspectMaskName = pexConfig.Field(
251  dtype=str,
252  doc="Name of mask plane to use for suspect pixels",
253  default="SUSPECT",
254  )
255  numEdgeSuspect = pexConfig.Field(
256  dtype=int,
257  doc="Number of edge pixels to be flagged as untrustworthy.",
258  default=0,
259  )
260 
261  # Initial masking options.
262  doSetBadRegions = pexConfig.Field(
263  dtype=bool,
264  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
265  default=True,
266  )
267  badStatistic = pexConfig.ChoiceField(
268  dtype=str,
269  doc="How to estimate the average value for BAD regions.",
270  default='MEANCLIP',
271  allowed={
272  "MEANCLIP": "Correct using the (clipped) mean of good data",
273  "MEDIAN": "Correct using the median of the good data",
274  },
275  )
276 
277  # Overscan subtraction configuration.
278  doOverscan = pexConfig.Field(
279  dtype=bool,
280  doc="Do overscan subtraction?",
281  default=True,
282  )
283  overscanFitType = pexConfig.ChoiceField(
284  dtype=str,
285  doc="The method for fitting the overscan bias level.",
286  default='MEDIAN',
287  allowed={
288  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
289  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
290  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
291  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
292  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
293  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
294  "MEAN": "Correct using the mean of the overscan region",
295  "MEANCLIP": "Correct using a clipped mean of the overscan region",
296  "MEDIAN": "Correct using the median of the overscan region",
297  },
298  )
299  overscanOrder = pexConfig.Field(
300  dtype=int,
301  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
302  "or number of spline knots if overscan fit type is a spline."),
303  default=1,
304  )
305  overscanNumSigmaClip = pexConfig.Field(
306  dtype=float,
307  doc="Rejection threshold (sigma) for collapsing overscan before fit",
308  default=3.0,
309  )
310  overscanIsInt = pexConfig.Field(
311  dtype=bool,
312  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
313  default=True,
314  )
315  overscanNumLeadingColumnsToSkip = pexConfig.Field(
316  dtype=int,
317  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
318  default=0,
319  )
320  overscanNumTrailingColumnsToSkip = pexConfig.Field(
321  dtype=int,
322  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
323  default=0,
324  )
325  overscanMaxDev = pexConfig.Field(
326  dtype=float,
327  doc="Maximum deviation from the median for overscan",
328  default=1000.0, check=lambda x: x > 0
329  )
330  overscanBiasJump = pexConfig.Field(
331  dtype=bool,
332  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
333  default=False,
334  )
335  overscanBiasJumpKeyword = pexConfig.Field(
336  dtype=str,
337  doc="Header keyword containing information about devices.",
338  default="NO_SUCH_KEY",
339  )
340  overscanBiasJumpDevices = pexConfig.ListField(
341  dtype=str,
342  doc="List of devices that need piecewise overscan correction.",
343  default=(),
344  )
345  overscanBiasJumpLocation = pexConfig.Field(
346  dtype=int,
347  doc="Location of bias jump along y-axis.",
348  default=0,
349  )
350 
351  # Amplifier to CCD assembly configuration
352  doAssembleCcd = pexConfig.Field(
353  dtype=bool,
354  default=True,
355  doc="Assemble amp-level exposures into a ccd-level exposure?"
356  )
357  assembleCcd = pexConfig.ConfigurableField(
358  target=AssembleCcdTask,
359  doc="CCD assembly task",
360  )
361 
362  # General calibration configuration.
363  doAssembleIsrExposures = pexConfig.Field(
364  dtype=bool,
365  default=False,
366  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
367  )
368  doTrimToMatchCalib = pexConfig.Field(
369  dtype=bool,
370  default=False,
371  doc="Trim raw data to match calibration bounding boxes?"
372  )
373 
374  # Bias subtraction.
375  doBias = pexConfig.Field(
376  dtype=bool,
377  doc="Apply bias frame correction?",
378  default=True,
379  )
380  biasDataProductName = pexConfig.Field(
381  dtype=str,
382  doc="Name of the bias data product",
383  default="bias",
384  )
385 
386  # Variance construction
387  doVariance = pexConfig.Field(
388  dtype=bool,
389  doc="Calculate variance?",
390  default=True
391  )
392  gain = pexConfig.Field(
393  dtype=float,
394  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
395  default=float("NaN"),
396  )
397  readNoise = pexConfig.Field(
398  dtype=float,
399  doc="The read noise to use if no Detector is present in the Exposure",
400  default=0.0,
401  )
402  doEmpiricalReadNoise = pexConfig.Field(
403  dtype=bool,
404  default=False,
405  doc="Calculate empirical read noise instead of value from AmpInfo data?"
406  )
407 
408  # Linearization.
409  doLinearize = pexConfig.Field(
410  dtype=bool,
411  doc="Correct for nonlinearity of the detector's response?",
412  default=True,
413  )
414 
415  # Crosstalk.
416  doCrosstalk = pexConfig.Field(
417  dtype=bool,
418  doc="Apply intra-CCD crosstalk correction?",
419  default=False,
420  )
421  doCrosstalkBeforeAssemble = pexConfig.Field(
422  dtype=bool,
423  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
424  default=True,
425  )
426  crosstalk = pexConfig.ConfigurableField(
427  target=CrosstalkTask,
428  doc="Intra-CCD crosstalk correction",
429  )
430 
431  # Masking options.
432  doDefect = pexConfig.Field(
433  dtype=bool,
434  doc="Apply correction for CCD defects, e.g. hot pixels?",
435  default=True,
436  )
437  numEdgeSuspect = pexConfig.Field(
438  dtype=int,
439  doc="Number of edge pixels to be flagged as untrustworthy.",
440  default=0,
441  )
442  doNanMasking = pexConfig.Field(
443  dtype=bool,
444  doc="Mask NAN pixels?",
445  default=True,
446  )
447  doWidenSaturationTrails = pexConfig.Field(
448  dtype=bool,
449  doc="Widen bleed trails based on their width?",
450  default=True
451  )
452 
453  # Brighter-Fatter correction.
454  doBrighterFatter = pexConfig.Field(
455  dtype=bool,
456  default=False,
457  doc="Apply the brighter fatter correction"
458  )
459  brighterFatterLevel = pexConfig.ChoiceField(
460  dtype=str,
461  default="DETECTOR",
462  doc="The level at which to correct for brighter-fatter.",
463  allowed={
464  "AMP": "Every amplifier treated separately.",
465  "DETECTOR": "One kernel per detector",
466  }
467  )
468  brighterFatterKernelFile = pexConfig.Field(
469  dtype=str,
470  default='',
471  doc="Kernel file used for the brighter fatter correction"
472  )
473  brighterFatterMaxIter = pexConfig.Field(
474  dtype=int,
475  default=10,
476  doc="Maximum number of iterations for the brighter fatter correction"
477  )
478  brighterFatterThreshold = pexConfig.Field(
479  dtype=float,
480  default=1000,
481  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
482  " absolute value of the difference between the current corrected image and the one"
483  " from the previous iteration summed over all the pixels."
484  )
485  brighterFatterApplyGain = pexConfig.Field(
486  dtype=bool,
487  default=True,
488  doc="Should the gain be applied when applying the brighter fatter correction?"
489  )
490 
491  # Dark subtraction.
492  doDark = pexConfig.Field(
493  dtype=bool,
494  doc="Apply dark frame correction?",
495  default=True,
496  )
497  darkDataProductName = pexConfig.Field(
498  dtype=str,
499  doc="Name of the dark data product",
500  default="dark",
501  )
502 
503  # Camera-specific stray light removal.
504  doStrayLight = pexConfig.Field(
505  dtype=bool,
506  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
507  default=False,
508  )
509  strayLight = pexConfig.ConfigurableField(
510  target=StrayLightTask,
511  doc="y-band stray light correction"
512  )
513 
514  # Flat correction.
515  doFlat = pexConfig.Field(
516  dtype=bool,
517  doc="Apply flat field correction?",
518  default=True,
519  )
520  flatDataProductName = pexConfig.Field(
521  dtype=str,
522  doc="Name of the flat data product",
523  default="flat",
524  )
525  flatScalingType = pexConfig.ChoiceField(
526  dtype=str,
527  doc="The method for scaling the flat on the fly.",
528  default='USER',
529  allowed={
530  "USER": "Scale by flatUserScale",
531  "MEAN": "Scale by the inverse of the mean",
532  "MEDIAN": "Scale by the inverse of the median",
533  },
534  )
535  flatUserScale = pexConfig.Field(
536  dtype=float,
537  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
538  default=1.0,
539  )
540  doTweakFlat = pexConfig.Field(
541  dtype=bool,
542  doc="Tweak flats to match observed amplifier ratios?",
543  default=False
544  )
545 
546  # Amplifier normalization based on gains instead of using flats configuration.
547  doApplyGains = pexConfig.Field(
548  dtype=bool,
549  doc="Correct the amplifiers for their gains instead of applying flat correction",
550  default=False,
551  )
552  normalizeGains = pexConfig.Field(
553  dtype=bool,
554  doc="Normalize all the amplifiers in each CCD to have the same median value.",
555  default=False,
556  )
557 
558  # Fringe correction.
559  doFringe = pexConfig.Field(
560  dtype=bool,
561  doc="Apply fringe correction?",
562  default=True,
563  )
564  fringe = pexConfig.ConfigurableField(
565  target=FringeTask,
566  doc="Fringe subtraction task",
567  )
568  fringeAfterFlat = pexConfig.Field(
569  dtype=bool,
570  doc="Do fringe subtraction after flat-fielding?",
571  default=True,
572  )
573 
574  # Distortion model application.
575  doAddDistortionModel = pexConfig.Field(
576  dtype=bool,
577  doc="Apply a distortion model based on camera geometry to the WCS?",
578  default=True,
579  )
580 
581  # Initial CCD-level background statistics options.
582  doMeasureBackground = pexConfig.Field(
583  dtype=bool,
584  doc="Measure the background level on the reduced image?",
585  default=False,
586  )
587 
588  # Camera-specific masking configuration.
589  doCameraSpecificMasking = pexConfig.Field(
590  dtype=bool,
591  doc="Mask camera-specific bad regions?",
592  default=False,
593  )
594  masking = pexConfig.ConfigurableField(
595  target=MaskingTask,
596  doc="Masking task."
597  )
598 
599  # Interpolation options.
600 
601  doInterpolate = pexConfig.Field(
602  dtype=bool,
603  doc="Interpolate masked pixels?",
604  default=True,
605  )
606  doSaturationInterpolation = pexConfig.Field(
607  dtype=bool,
608  doc="Perform interpolation over pixels masked as saturated?"
609  " NB: This is independent of doSaturation; if that is False this plane"
610  " will likely be blank, resulting in a no-op here.",
611  default=True,
612  )
613  doNanInterpolation = pexConfig.Field(
614  dtype=bool,
615  doc="Perform interpolation over pixels masked as NaN?"
616  " NB: This is independent of doNanMasking; if that is False this plane"
617  " will likely be blank, resulting in a no-op here.",
618  default=True,
619  )
620  doNanInterpAfterFlat = pexConfig.Field(
621  dtype=bool,
622  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
623  "also have to interpolate them before flat-fielding."),
624  default=False,
625  )
626  maskListToInterpolate = pexConfig.ListField(
627  dtype=str,
628  doc="List of mask planes that should be interpolated.",
629  default=['SAT', 'BAD', 'UNMASKEDNAN'],
630  )
631  doSaveInterpPixels = pexConfig.Field(
632  dtype=bool,
633  doc="Save a copy of the pre-interpolated pixel values?",
634  default=False,
635  )
636 
637  # Default photometric calibration options.
638  fluxMag0T1 = pexConfig.DictField(
639  keytype=str,
640  itemtype=float,
641  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
642  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
643  ))
644  )
645  defaultFluxMag0T1 = pexConfig.Field(
646  dtype=float,
647  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
648  default=pow(10.0, 0.4*28.0)
649  )
650 
651  # Vignette correction configuration.
652  doVignette = pexConfig.Field(
653  dtype=bool,
654  doc="Apply vignetting parameters?",
655  default=False,
656  )
657  vignette = pexConfig.ConfigurableField(
658  target=VignetteTask,
659  doc="Vignetting task.",
660  )
661 
662  # Transmission curve configuration.
663  doAttachTransmissionCurve = pexConfig.Field(
664  dtype=bool,
665  default=False,
666  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
667  )
668  doUseOpticsTransmission = pexConfig.Field(
669  dtype=bool,
670  default=True,
671  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
672  )
673  doUseFilterTransmission = pexConfig.Field(
674  dtype=bool,
675  default=True,
676  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
677  )
678  doUseSensorTransmission = pexConfig.Field(
679  dtype=bool,
680  default=True,
681  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
682  )
683  doUseAtmosphereTransmission = pexConfig.Field(
684  dtype=bool,
685  default=True,
686  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
687  )
688 
689  # Illumination correction.
690  doIlluminationCorrection = pexConfig.Field(
691  dtype=bool,
692  default=False,
693  doc="Perform illumination correction?"
694  )
695  illuminationCorrectionDataProductName = pexConfig.Field(
696  dtype=str,
697  doc="Name of the illumination correction data product.",
698  default="illumcor",
699  )
700  illumScale = pexConfig.Field(
701  dtype=float,
702  doc="Scale factor for the illumination correction.",
703  default=1.0,
704  )
705  illumFilters = pexConfig.ListField(
706  dtype=str,
707  default=[],
708  doc="Only perform illumination correction for these filters."
709  )
710 
711  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
712  doWrite = pexConfig.Field(
713  dtype=bool,
714  doc="Persist postISRCCD?",
715  default=True,
716  )
717 
718  def validate(self):
719  super().validate()
720  if self.doFlat and self.doApplyGains:
721  raise ValueError("You may not specify both doFlat and doApplyGains")
722  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
723  self.config.maskListToInterpolate.append("SAT")
724  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
725  self.config.maskListToInterpolate.append("UNMASKEDNAN")
726 
727 
728 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
729  """Apply common instrument signature correction algorithms to a raw frame.
730 
731  The process for correcting imaging data is very similar from
732  camera to camera. This task provides a vanilla implementation of
733  doing these corrections, including the ability to turn certain
734  corrections off if they are not needed. The inputs to the primary
735  method, `run()`, are a raw exposure to be corrected and the
736  calibration data products. The raw input is a single chip sized
737  mosaic of all amps including overscans and other non-science
738  pixels. The method `runDataRef()` identifies and defines the
739  calibration data products, and is intended for use by a
740  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
741  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
742  subclassed for different camera, although the most camera specific
743  methods have been split into subtasks that can be redirected
744  appropriately.
745 
746  The __init__ method sets up the subtasks for ISR processing, using
747  the defaults from `lsst.ip.isr`.
748 
749  Parameters
750  ----------
751  args : `list`
752  Positional arguments passed to the Task constructor. None used at this time.
753  kwargs : `dict`, optional
754  Keyword arguments passed on to the Task constructor. None used at this time.
755  """
756  ConfigClass = IsrTaskConfig
757  _DefaultName = "isr"
758 
759  def __init__(self, **kwargs):
760  super().__init__(**kwargs)
761  self.makeSubtask("assembleCcd")
762  self.makeSubtask("crosstalk")
763  self.makeSubtask("strayLight")
764  self.makeSubtask("fringe")
765  self.makeSubtask("masking")
766  self.makeSubtask("vignette")
767 
768  @classmethod
769  def getInputDatasetTypes(cls, config):
770  inputTypeDict = super().getInputDatasetTypes(config)
771 
772  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
773  # need because the configuration tells us we will not be bothering with the
774  # correction that uses that IDT.
775  if config.doBias is not True:
776  inputTypeDict.pop("bias", None)
777  if config.doLinearize is not True:
778  inputTypeDict.pop("linearizer", None)
779  if config.doCrosstalk is not True:
780  inputTypeDict.pop("crosstalkSources", None)
781  if config.doBrighterFatter is not True:
782  inputTypeDict.pop("bfKernel", None)
783  if config.doDefect is not True:
784  inputTypeDict.pop("defects", None)
785  if config.doDark is not True:
786  inputTypeDict.pop("dark", None)
787  if config.doFlat is not True:
788  inputTypeDict.pop("flat", None)
789  if config.doAttachTransmissionCurve is not True:
790  inputTypeDict.pop("opticsTransmission", None)
791  inputTypeDict.pop("filterTransmission", None)
792  inputTypeDict.pop("sensorTransmission", None)
793  inputTypeDict.pop("atmosphereTransmission", None)
794  if config.doUseOpticsTransmission is not True:
795  inputTypeDict.pop("opticsTransmission", None)
796  if config.doUseFilterTransmission is not True:
797  inputTypeDict.pop("filterTransmission", None)
798  if config.doUseSensorTransmission is not True:
799  inputTypeDict.pop("sensorTransmission", None)
800  if config.doUseAtmosphereTransmission is not True:
801  inputTypeDict.pop("atmosphereTransmission", None)
802  if config.doIlluminationCorrection is not True:
803  inputTypeDict.pop("illumMaskedImage", None)
804 
805  return inputTypeDict
806 
807  @classmethod
808  def getOutputDatasetTypes(cls, config):
809  outputTypeDict = super().getOutputDatasetTypes(config)
810 
811  if config.qa.doThumbnailOss is not True:
812  outputTypeDict.pop("outputOssThumbnail", None)
813  if config.qa.doThumbnailFlattened is not True:
814  outputTypeDict.pop("outputFlattenedThumbnail", None)
815  if config.doWrite is not True:
816  outputTypeDict.pop("outputExposure", None)
817 
818  return outputTypeDict
819 
820  @classmethod
821  def getPrerequisiteDatasetTypes(cls, config):
822  # Input calibration datasets should not constrain the QuantumGraph
823  # (it'd be confusing if not having flats just silently resulted in no
824  # data being processed). Our nomenclature for that is that these are
825  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
826  names = set(cls.getInputDatasetTypes(config))
827  names.remove("ccdExposure")
828  return names
829 
830  @classmethod
831  def getPerDatasetTypeDimensions(cls, config):
832  # Input calibration datasets of different types (i.e. flat, bias) need
833  # not have the same validity range. That makes calibration_label
834  # (which maps directly to a validity range) a "per-DatasetType
835  # dimension".
836  return frozenset(["calibration_label"])
837 
838  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
839  try:
840  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
841  except Exception as e:
842  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
843 
844  inputData['isGen3'] = True
845 
846  if self.config.doLinearize is True:
847  if 'linearizer' not in inputData.keys():
848  detector = inputData['camera'][inputData['detectorNum']]
849  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
850  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
851 
852  if inputData['defects'] is not None:
853  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
854  # masking expects a list of defects defined by their bounding box
855  if not isinstance(inputData["defects"], Defects):
856  inputData["defects"] = Defects.fromTable(inputData["defects"])
857 
858  # Broken: DM-17169
859  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
860  # fixed for non-HSC cameras in the future.
861  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
862  # if self.config.doCrosstalk else None)
863 
864  # Broken: DM-17152
865  # Fringes are not tested to be handled correctly by Gen3 butler.
866  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
867  # assembler=self.assembleCcd
868  # if self.config.doAssembleIsrExposures else None)
869  # if self.config.doFringe and
870  # self.fringe.checkFilter(inputData['ccdExposure'])
871  # else pipeBase.Struct(fringes=None))
872 
873  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
874 
875  def makeDatasetType(self, dsConfig):
876  return super().makeDatasetType(dsConfig)
877 
878  def readIsrData(self, dataRef, rawExposure):
879  """!Retrieve necessary frames for instrument signature removal.
880 
881  Pre-fetching all required ISR data products limits the IO
882  required by the ISR. Any conflict between the calibration data
883  available and that needed for ISR is also detected prior to
884  doing processing, allowing it to fail quickly.
885 
886  Parameters
887  ----------
888  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
889  Butler reference of the detector data to be processed
890  rawExposure : `afw.image.Exposure`
891  The raw exposure that will later be corrected with the
892  retrieved calibration data; should not be modified in this
893  method.
894 
895  Returns
896  -------
897  result : `lsst.pipe.base.Struct`
898  Result struct with components (which may be `None`):
899  - ``bias``: bias calibration frame (`afw.image.Exposure`)
900  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
901  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
902  - ``dark``: dark calibration frame (`afw.image.Exposure`)
903  - ``flat``: flat calibration frame (`afw.image.Exposure`)
904  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
905  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
906  - ``fringes``: `lsst.pipe.base.Struct` with components:
907  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
908  - ``seed``: random seed derived from the ccdExposureId for random
909  number generator (`uint32`).
910  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
911  A ``TransmissionCurve`` that represents the throughput of the optics,
912  to be evaluated in focal-plane coordinates.
913  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
914  A ``TransmissionCurve`` that represents the throughput of the filter
915  itself, to be evaluated in focal-plane coordinates.
916  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
917  A ``TransmissionCurve`` that represents the throughput of the sensor
918  itself, to be evaluated in post-assembly trimmed detector coordinates.
919  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
920  A ``TransmissionCurve`` that represents the throughput of the
921  atmosphere, assumed to be spatially constant.
922  - ``strayLightData`` : `object`
923  An opaque object containing calibration information for
924  stray-light correction. If `None`, no correction will be
925  performed.
926  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
927 
928  Raises
929  ------
930  NotImplementedError :
931  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
932  """
933  ccd = rawExposure.getDetector()
934  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
935  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
936  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
937  if self.config.doBias else None)
938  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
939  linearizer = (dataRef.get("linearizer", immediate=True)
940  if self.doLinearize(ccd) else None)
941  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
942  if self.config.doCrosstalk else None)
943  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
944  if self.config.doDark else None)
945  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
946  if self.config.doFlat else None)
947 
948  brighterFatterKernel = None
949  if self.config.doBrighterFatter is True:
950 
951  # Use the new-style cp_pipe version of the kernel is it exists.
952  try:
953  brighterFatterKernel = dataRef.get("brighterFatterKernel")
954  except NoResults:
955  # Fall back to the old-style numpy-ndarray style kernel if necessary.
956  try:
957  brighterFatterKernel = dataRef.get("bfKernel")
958  except NoResults:
959  brighterFatterKernel = None
960  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
961  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
962  # this detector, or raise an error.
963  if self.config.brighterFatterLevel == 'DETECTOR':
964  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
965  else:
966  # TODO DM-15631 for implementing this
967  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
968 
969  defectList = (dataRef.get("defects")
970  if self.config.doDefect else None)
971  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
972  if self.config.doAssembleIsrExposures else None)
973  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
974  else pipeBase.Struct(fringes=None))
975 
976  if self.config.doAttachTransmissionCurve:
977  opticsTransmission = (dataRef.get("transmission_optics")
978  if self.config.doUseOpticsTransmission else None)
979  filterTransmission = (dataRef.get("transmission_filter")
980  if self.config.doUseFilterTransmission else None)
981  sensorTransmission = (dataRef.get("transmission_sensor")
982  if self.config.doUseSensorTransmission else None)
983  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
984  if self.config.doUseAtmosphereTransmission else None)
985  else:
986  opticsTransmission = None
987  filterTransmission = None
988  sensorTransmission = None
989  atmosphereTransmission = None
990 
991  if self.config.doStrayLight:
992  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
993  else:
994  strayLightData = None
995 
996  illumMaskedImage = (self.getIsrExposure(dataRef,
997  self.config.illuminationCorrectionDataProductName).getMaskedImage()
998  if (self.config.doIlluminationCorrection and
999  filterName in self.config.illumFilters)
1000  else None)
1001 
1002  # Struct should include only kwargs to run()
1003  return pipeBase.Struct(bias=biasExposure,
1004  linearizer=linearizer,
1005  crosstalkSources=crosstalkSources,
1006  dark=darkExposure,
1007  flat=flatExposure,
1008  bfKernel=brighterFatterKernel,
1009  defects=defectList,
1010  fringes=fringeStruct,
1011  opticsTransmission=opticsTransmission,
1012  filterTransmission=filterTransmission,
1013  sensorTransmission=sensorTransmission,
1014  atmosphereTransmission=atmosphereTransmission,
1015  strayLightData=strayLightData,
1016  illumMaskedImage=illumMaskedImage
1017  )
1018 
1019  @pipeBase.timeMethod
1020  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1021  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1022  opticsTransmission=None, filterTransmission=None,
1023  sensorTransmission=None, atmosphereTransmission=None,
1024  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1025  isGen3=False,
1026  ):
1027  """!Perform instrument signature removal on an exposure.
1028 
1029  Steps included in the ISR processing, in order performed, are:
1030  - saturation and suspect pixel masking
1031  - overscan subtraction
1032  - CCD assembly of individual amplifiers
1033  - bias subtraction
1034  - variance image construction
1035  - linearization of non-linear response
1036  - crosstalk masking
1037  - brighter-fatter correction
1038  - dark subtraction
1039  - fringe correction
1040  - stray light subtraction
1041  - flat correction
1042  - masking of known defects and camera specific features
1043  - vignette calculation
1044  - appending transmission curve and distortion model
1045 
1046  Parameters
1047  ----------
1048  ccdExposure : `lsst.afw.image.Exposure`
1049  The raw exposure that is to be run through ISR. The
1050  exposure is modified by this method.
1051  camera : `lsst.afw.cameraGeom.Camera`, optional
1052  The camera geometry for this exposure. Used to select the
1053  distortion model appropriate for this data.
1054  bias : `lsst.afw.image.Exposure`, optional
1055  Bias calibration frame.
1056  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1057  Functor for linearization.
1058  crosstalkSources : `list`, optional
1059  List of possible crosstalk sources.
1060  dark : `lsst.afw.image.Exposure`, optional
1061  Dark calibration frame.
1062  flat : `lsst.afw.image.Exposure`, optional
1063  Flat calibration frame.
1064  bfKernel : `numpy.ndarray`, optional
1065  Brighter-fatter kernel.
1066  defects : `lsst.meas.algorithms.Defects`, optional
1067  List of defects.
1068  fringes : `lsst.pipe.base.Struct`, optional
1069  Struct containing the fringe correction data, with
1070  elements:
1071  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1072  - ``seed``: random seed derived from the ccdExposureId for random
1073  number generator (`uint32`)
1074  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1075  A ``TransmissionCurve`` that represents the throughput of the optics,
1076  to be evaluated in focal-plane coordinates.
1077  filterTransmission : `lsst.afw.image.TransmissionCurve`
1078  A ``TransmissionCurve`` that represents the throughput of the filter
1079  itself, to be evaluated in focal-plane coordinates.
1080  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1081  A ``TransmissionCurve`` that represents the throughput of the sensor
1082  itself, to be evaluated in post-assembly trimmed detector coordinates.
1083  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1084  A ``TransmissionCurve`` that represents the throughput of the
1085  atmosphere, assumed to be spatially constant.
1086  detectorNum : `int`, optional
1087  The integer number for the detector to process.
1088  isGen3 : bool, optional
1089  Flag this call to run() as using the Gen3 butler environment.
1090  strayLightData : `object`, optional
1091  Opaque object containing calibration information for stray-light
1092  correction. If `None`, no correction will be performed.
1093  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1094  Illumination correction image.
1095 
1096  Returns
1097  -------
1098  result : `lsst.pipe.base.Struct`
1099  Result struct with component:
1100  - ``exposure`` : `afw.image.Exposure`
1101  The fully ISR corrected exposure.
1102  - ``outputExposure`` : `afw.image.Exposure`
1103  An alias for `exposure`
1104  - ``ossThumb`` : `numpy.ndarray`
1105  Thumbnail image of the exposure after overscan subtraction.
1106  - ``flattenedThumb`` : `numpy.ndarray`
1107  Thumbnail image of the exposure after flat-field correction.
1108 
1109  Raises
1110  ------
1111  RuntimeError
1112  Raised if a configuration option is set to True, but the
1113  required calibration data has not been specified.
1114 
1115  Notes
1116  -----
1117  The current processed exposure can be viewed by setting the
1118  appropriate lsstDebug entries in the `debug.display`
1119  dictionary. The names of these entries correspond to some of
1120  the IsrTaskConfig Boolean options, with the value denoting the
1121  frame to use. The exposure is shown inside the matching
1122  option check and after the processing of that step has
1123  finished. The steps with debug points are:
1124 
1125  doAssembleCcd
1126  doBias
1127  doCrosstalk
1128  doBrighterFatter
1129  doDark
1130  doFringe
1131  doStrayLight
1132  doFlat
1133 
1134  In addition, setting the "postISRCCD" entry displays the
1135  exposure after all ISR processing has finished.
1136 
1137  """
1138 
1139  if isGen3 is True:
1140  # Gen3 currently cannot automatically do configuration overrides.
1141  # DM-15257 looks to discuss this issue.
1142 
1143  self.config.doFringe = False
1144 
1145  # Configure input exposures;
1146  if detectorNum is None:
1147  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1148 
1149  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1150  bias = self.ensureExposure(bias, camera, detectorNum)
1151  dark = self.ensureExposure(dark, camera, detectorNum)
1152  flat = self.ensureExposure(flat, camera, detectorNum)
1153  else:
1154  if isinstance(ccdExposure, ButlerDataRef):
1155  return self.runDataRef(ccdExposure)
1156 
1157  ccd = ccdExposure.getDetector()
1158  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1159 
1160  if not ccd:
1161  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1162  ccd = [FakeAmp(ccdExposure, self.config)]
1163 
1164  # Validate Input
1165  if self.config.doBias and bias is None:
1166  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1167  if self.doLinearize(ccd) and linearizer is None:
1168  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1169  if self.config.doBrighterFatter and bfKernel is None:
1170  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1171  if self.config.doDark and dark is None:
1172  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1173  if self.config.doFlat and flat is None:
1174  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1175  if self.config.doDefect and defects is None:
1176  raise RuntimeError("Must supply defects if config.doDefect=True.")
1177  if self.config.doAddDistortionModel and camera is None:
1178  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1179  if (self.config.doFringe and filterName in self.fringe.config.filters and
1180  fringes.fringes is None):
1181  # The `fringes` object needs to be a pipeBase.Struct, as
1182  # we use it as a `dict` for the parameters of
1183  # `FringeTask.run()`. The `fringes.fringes` `list` may
1184  # not be `None` if `doFringe=True`. Otherwise, raise.
1185  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1186  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1187  illumMaskedImage is None):
1188  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1189 
1190  # Begin ISR processing.
1191  if self.config.doConvertIntToFloat:
1192  self.log.info("Converting exposure to floating point values.")
1193  ccdExposure = self.convertIntToFloat(ccdExposure)
1194 
1195  # Amplifier level processing.
1196  overscans = []
1197  for amp in ccd:
1198  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1199  if ccdExposure.getBBox().contains(amp.getBBox()):
1200  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1201  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1202 
1203  if self.config.doOverscan and not badAmp:
1204  # Overscan correction on amp-by-amp basis.
1205  overscanResults = self.overscanCorrection(ccdExposure, amp)
1206  self.log.debug(f"Corrected overscan for amplifier {amp.getName()}.")
1207  if overscanResults is not None and \
1208  self.config.qa is not None and self.config.qa.saveStats is True:
1209  if isinstance(overscanResults.overscanFit, float):
1210  qaMedian = overscanResults.overscanFit
1211  qaStdev = float("NaN")
1212  else:
1213  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1214  afwMath.MEDIAN | afwMath.STDEVCLIP)
1215  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1216  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1217 
1218  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1219  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1220  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1221  (amp.getName(), qaMedian, qaStdev))
1222  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1223  else:
1224  if badAmp:
1225  self.log.warn(f"Amplifier {amp.getName()} is bad.")
1226  overscanResults = None
1227 
1228  overscans.append(overscanResults if overscanResults is not None else None)
1229  else:
1230  self.log.info(f"Skipped OSCAN for {amp.getName()}.")
1231 
1232  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1233  self.log.info("Applying crosstalk correction.")
1234  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1235  self.debugView(ccdExposure, "doCrosstalk")
1236 
1237  if self.config.doAssembleCcd:
1238  self.log.info("Assembling CCD from amplifiers.")
1239  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1240 
1241  if self.config.expectWcs and not ccdExposure.getWcs():
1242  self.log.warn("No WCS found in input exposure.")
1243  self.debugView(ccdExposure, "doAssembleCcd")
1244 
1245  ossThumb = None
1246  if self.config.qa.doThumbnailOss:
1247  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1248 
1249  if self.config.doBias:
1250  self.log.info("Applying bias correction.")
1251  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1252  trimToFit=self.config.doTrimToMatchCalib)
1253  self.debugView(ccdExposure, "doBias")
1254 
1255  if self.config.doVariance:
1256  for amp, overscanResults in zip(ccd, overscans):
1257  if ccdExposure.getBBox().contains(amp.getBBox()):
1258  self.log.debug(f"Constructing variance map for amplifer {amp.getName()}.")
1259  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1260  if overscanResults is not None:
1261  self.updateVariance(ampExposure, amp,
1262  overscanImage=overscanResults.overscanImage)
1263  else:
1264  self.updateVariance(ampExposure, amp,
1265  overscanImage=None)
1266  if self.config.qa is not None and self.config.qa.saveStats is True:
1267  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1268  afwMath.MEDIAN | afwMath.STDEVCLIP)
1269  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1270  qaStats.getValue(afwMath.MEDIAN))
1271  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1272  qaStats.getValue(afwMath.STDEVCLIP))
1273  self.log.debug(" Variance stats for amplifer %s: %f +/- %f." %
1274  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1275  qaStats.getValue(afwMath.STDEVCLIP)))
1276 
1277  if self.doLinearize(ccd):
1278  self.log.info("Applying linearizer.")
1279  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1280 
1281  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1282  self.log.info("Applying crosstalk correction.")
1283  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1284  self.debugView(ccdExposure, "doCrosstalk")
1285 
1286  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1287  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1288  if self.config.doDefect:
1289  self.log.info("Masking defects.")
1290  self.maskDefect(ccdExposure, defects)
1291 
1292  if self.config.doNanMasking:
1293  self.log.info("Masking NAN value pixels.")
1294  self.maskNan(ccdExposure)
1295 
1296  if self.config.doWidenSaturationTrails:
1297  self.log.info("Widening saturation trails.")
1298  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1299 
1300  if self.config.doCameraSpecificMasking:
1301  self.log.info("Masking regions for camera specific reasons.")
1302  self.masking.run(ccdExposure)
1303 
1304  if self.config.doBrighterFatter:
1305  # We need to apply flats and darks before we can interpolate, and we
1306  # need to interpolate before we do B-F, but we do B-F without the
1307  # flats and darks applied so we can work in units of electrons or holes.
1308  # This context manager applies and then removes the darks and flats.
1309  #
1310  # We also do not want to interpolate values here, so operate on temporary
1311  # images so we can apply only the BF-correction and roll back the
1312  # interpolation.
1313  interpExp = ccdExposure.clone()
1314  with self.flatContext(interpExp, flat, dark):
1315  isrFunctions.interpolateFromMask(
1316  maskedImage=interpExp.getMaskedImage(),
1317  fwhm=self.config.fwhm,
1318  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1319  maskNameList=self.config.maskListToInterpolate
1320  )
1321  bfExp = interpExp.clone()
1322 
1323  self.log.info("Applying brighter fatter correction.")
1324  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1325  self.config.brighterFatterMaxIter,
1326  self.config.brighterFatterThreshold,
1327  self.config.brighterFatterApplyGain
1328  )
1329  if bfResults[1] == self.config.brighterFatterMaxIter:
1330  self.log.warn("Brighter fatter correction did not converge, final difference {bfResults[0]}.")
1331  else:
1332  self.log.info("Finished brighter fatter correction in {bfResults[1]} iterations.")
1333  image = ccdExposure.getMaskedImage().getImage()
1334  bfCorr = bfExp.getMaskedImage().getImage()
1335  bfCorr -= interpExp.getMaskedImage().getImage()
1336  image += bfCorr
1337 
1338  self.debugView(ccdExposure, "doBrighterFatter")
1339 
1340  if self.config.doDark:
1341  self.log.info("Applying dark correction.")
1342  self.darkCorrection(ccdExposure, dark)
1343  self.debugView(ccdExposure, "doDark")
1344 
1345  if self.config.doFringe and not self.config.fringeAfterFlat:
1346  self.log.info("Applying fringe correction before flat.")
1347  self.fringe.run(ccdExposure, **fringes.getDict())
1348  self.debugView(ccdExposure, "doFringe")
1349 
1350  if self.config.doStrayLight:
1351  if strayLightData is not None:
1352  self.log.info("Applying stray light correction.")
1353  self.strayLight.run(ccdExposure, strayLightData)
1354  self.debugView(ccdExposure, "doStrayLight")
1355  else:
1356  self.log.debug("Skipping stray light correction: no data found for this image.")
1357 
1358  if self.config.doFlat:
1359  self.log.info("Applying flat correction.")
1360  self.flatCorrection(ccdExposure, flat)
1361  self.debugView(ccdExposure, "doFlat")
1362 
1363  if self.config.doApplyGains:
1364  self.log.info("Applying gain correction instead of flat.")
1365  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1366 
1367  if self.config.doFringe and self.config.fringeAfterFlat:
1368  self.log.info("Applying fringe correction after flat.")
1369  self.fringe.run(ccdExposure, **fringes.getDict())
1370 
1371  if self.config.doVignette:
1372  self.log.info("Constructing Vignette polygon.")
1373  self.vignettePolygon = self.vignette.run(ccdExposure)
1374 
1375  if self.config.vignette.doWriteVignettePolygon:
1376  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1377 
1378  if self.config.doAttachTransmissionCurve:
1379  self.log.info("Adding transmission curves.")
1380  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1381  filterTransmission=filterTransmission,
1382  sensorTransmission=sensorTransmission,
1383  atmosphereTransmission=atmosphereTransmission)
1384 
1385  if self.config.doAddDistortionModel:
1386  self.log.info("Adding a distortion model to the WCS.")
1387  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1388 
1389  flattenedThumb = None
1390  if self.config.qa.doThumbnailFlattened:
1391  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1392 
1393  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1394  self.log.info("Performing illumination correction.")
1395  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1396  illumMaskedImage, illumScale=self.config.illumScale,
1397  trimToFit=self.config.doTrimToMatchCalib)
1398 
1399  preInterpExp = None
1400  if self.config.doSaveInterpPixels:
1401  preInterpExp = ccdExposure.clone()
1402 
1403  # Reset and interpolate bad pixels.
1404  #
1405  # Large contiguous bad regions (which should have the BAD mask
1406  # bit set) should have their values set to the image median.
1407  # This group should include defects and bad amplifiers. As the
1408  # area covered by these defects are large, there's little
1409  # reason to expect that interpolation would provide a more
1410  # useful value.
1411  #
1412  # Smaller defects can be safely interpolated after the larger
1413  # regions have had their pixel values reset. This ensures
1414  # that the remaining defects adjacent to bad amplifiers (as an
1415  # example) do not attempt to interpolate extreme values.
1416  if self.config.doSetBadRegions:
1417  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1418  if badPixelCount > 0:
1419  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1420 
1421  if self.config.doInterpolate:
1422  self.log.info("Interpolating masked pixels.")
1423  isrFunctions.interpolateFromMask(
1424  maskedImage=ccdExposure.getMaskedImage(),
1425  fwhm=self.config.fwhm,
1426  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1427  maskNameList=list(self.config.maskListToInterpolate)
1428  )
1429 
1430  self.roughZeroPoint(ccdExposure)
1431 
1432  if self.config.doMeasureBackground:
1433  self.log.info("Measuring background level.")
1434  self.measureBackground(ccdExposure, self.config.qa)
1435 
1436  if self.config.qa is not None and self.config.qa.saveStats is True:
1437  for amp in ccd:
1438  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1439  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1440  afwMath.MEDIAN | afwMath.STDEVCLIP)
1441  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1442  qaStats.getValue(afwMath.MEDIAN))
1443  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1444  qaStats.getValue(afwMath.STDEVCLIP))
1445  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1446  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1447  qaStats.getValue(afwMath.STDEVCLIP)))
1448 
1449  self.debugView(ccdExposure, "postISRCCD")
1450 
1451  return pipeBase.Struct(
1452  exposure=ccdExposure,
1453  ossThumb=ossThumb,
1454  flattenedThumb=flattenedThumb,
1455 
1456  preInterpolatedExposure=preInterpExp,
1457  outputExposure=ccdExposure,
1458  outputOssThumbnail=ossThumb,
1459  outputFlattenedThumbnail=flattenedThumb,
1460  )
1461 
1462  @pipeBase.timeMethod
1463  def runDataRef(self, sensorRef):
1464  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1465 
1466  This method contains the `CmdLineTask` interface to the ISR
1467  processing. All IO is handled here, freeing the `run()` method
1468  to manage only pixel-level calculations. The steps performed
1469  are:
1470  - Read in necessary detrending/isr/calibration data.
1471  - Process raw exposure in `run()`.
1472  - Persist the ISR-corrected exposure as "postISRCCD" if
1473  config.doWrite=True.
1474 
1475  Parameters
1476  ----------
1477  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1478  DataRef of the detector data to be processed
1479 
1480  Returns
1481  -------
1482  result : `lsst.pipe.base.Struct`
1483  Result struct with component:
1484  - ``exposure`` : `afw.image.Exposure`
1485  The fully ISR corrected exposure.
1486 
1487  Raises
1488  ------
1489  RuntimeError
1490  Raised if a configuration option is set to True, but the
1491  required calibration data does not exist.
1492 
1493  """
1494  self.log.info("Performing ISR on sensor %s." % (sensorRef.dataId))
1495 
1496  ccdExposure = sensorRef.get(self.config.datasetType)
1497 
1498  camera = sensorRef.get("camera")
1499  if camera is None and self.config.doAddDistortionModel:
1500  raise RuntimeError("config.doAddDistortionModel is True "
1501  "but could not get a camera from the butler.")
1502  isrData = self.readIsrData(sensorRef, ccdExposure)
1503 
1504  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1505 
1506  if self.config.doWrite:
1507  sensorRef.put(result.exposure, "postISRCCD")
1508  if result.preInterpolatedExposure is not None:
1509  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1510  if result.ossThumb is not None:
1511  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1512  if result.flattenedThumb is not None:
1513  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1514 
1515  return result
1516 
1517  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1518  """!Retrieve a calibration dataset for removing instrument signature.
1519 
1520  Parameters
1521  ----------
1522 
1523  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1524  DataRef of the detector data to find calibration datasets
1525  for.
1526  datasetType : `str`
1527  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1528  immediate : `Bool`
1529  If True, disable butler proxies to enable error handling
1530  within this routine.
1531 
1532  Returns
1533  -------
1534  exposure : `lsst.afw.image.Exposure`
1535  Requested calibration frame.
1536 
1537  Raises
1538  ------
1539  RuntimeError
1540  Raised if no matching calibration frame can be found.
1541  """
1542  try:
1543  exp = dataRef.get(datasetType, immediate=immediate)
1544  except Exception as exc1:
1545  if not self.config.fallbackFilterName:
1546  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1547  try:
1548  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1549  except Exception as exc2:
1550  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1551  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1552  self.log.warn("Using fallback calibration from filter %s." % self.config.fallbackFilterName)
1553 
1554  if self.config.doAssembleIsrExposures:
1555  exp = self.assembleCcd.assembleCcd(exp)
1556  return exp
1557 
1558  def ensureExposure(self, inputExp, camera, detectorNum):
1559  """Ensure that the data returned by Butler is a fully constructed exposure.
1560 
1561  ISR requires exposure-level image data for historical reasons, so if we did
1562  not recieve that from Butler, construct it from what we have, modifying the
1563  input in place.
1564 
1565  Parameters
1566  ----------
1567  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1568  `lsst.afw.image.ImageF`
1569  The input data structure obtained from Butler.
1570  camera : `lsst.afw.cameraGeom.camera`
1571  The camera associated with the image. Used to find the appropriate
1572  detector.
1573  detectorNum : `int`
1574  The detector this exposure should match.
1575 
1576  Returns
1577  -------
1578  inputExp : `lsst.afw.image.Exposure`
1579  The re-constructed exposure, with appropriate detector parameters.
1580 
1581  Raises
1582  ------
1583  TypeError
1584  Raised if the input data cannot be used to construct an exposure.
1585  """
1586  if isinstance(inputExp, afwImage.DecoratedImageU):
1587  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1588  elif isinstance(inputExp, afwImage.ImageF):
1589  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1590  elif isinstance(inputExp, afwImage.MaskedImageF):
1591  inputExp = afwImage.makeExposure(inputExp)
1592  elif isinstance(inputExp, afwImage.Exposure):
1593  pass
1594  elif inputExp is None:
1595  # Assume this will be caught by the setup if it is a problem.
1596  return inputExp
1597  else:
1598  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}.")
1599 
1600  if inputExp.getDetector() is None:
1601  inputExp.setDetector(camera[detectorNum])
1602 
1603  return inputExp
1604 
1605  def convertIntToFloat(self, exposure):
1606  """Convert exposure image from uint16 to float.
1607 
1608  If the exposure does not need to be converted, the input is
1609  immediately returned. For exposures that are converted to use
1610  floating point pixels, the variance is set to unity and the
1611  mask to zero.
1612 
1613  Parameters
1614  ----------
1615  exposure : `lsst.afw.image.Exposure`
1616  The raw exposure to be converted.
1617 
1618  Returns
1619  -------
1620  newexposure : `lsst.afw.image.Exposure`
1621  The input ``exposure``, converted to floating point pixels.
1622 
1623  Raises
1624  ------
1625  RuntimeError
1626  Raised if the exposure type cannot be converted to float.
1627 
1628  """
1629  if isinstance(exposure, afwImage.ExposureF):
1630  # Nothing to be done
1631  self.log.debug("Exposure already of type float.")
1632  return exposure
1633  if not hasattr(exposure, "convertF"):
1634  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1635 
1636  newexposure = exposure.convertF()
1637  newexposure.variance[:] = 1
1638  newexposure.mask[:] = 0x0
1639 
1640  return newexposure
1641 
1642  def maskAmplifier(self, ccdExposure, amp, defects):
1643  """Identify bad amplifiers, saturated and suspect pixels.
1644 
1645  Parameters
1646  ----------
1647  ccdExposure : `lsst.afw.image.Exposure`
1648  Input exposure to be masked.
1649  amp : `lsst.afw.table.AmpInfoCatalog`
1650  Catalog of parameters defining the amplifier on this
1651  exposure to mask.
1652  defects : `lsst.meas.algorithms.Defects`
1653  List of defects. Used to determine if the entire
1654  amplifier is bad.
1655 
1656  Returns
1657  -------
1658  badAmp : `Bool`
1659  If this is true, the entire amplifier area is covered by
1660  defects and unusable.
1661 
1662  """
1663  maskedImage = ccdExposure.getMaskedImage()
1664 
1665  badAmp = False
1666 
1667  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1668  # comparison with current defects definition.
1669  if defects is not None:
1670  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1671 
1672  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1673  # association with pixels in current ccdExposure).
1674  if badAmp:
1675  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1676  afwImage.PARENT)
1677  maskView = dataView.getMask()
1678  maskView |= maskView.getPlaneBitMask("BAD")
1679  del maskView
1680  return badAmp
1681 
1682  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1683  # Saturation and suspect pixels can be masked now, though.
1684  limits = dict()
1685  if self.config.doSaturation and not badAmp:
1686  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1687  if self.config.doSuspect and not badAmp:
1688  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1689  if math.isfinite(self.config.saturation):
1690  limits.update({self.config.saturatedMaskName: self.config.saturation})
1691 
1692  for maskName, maskThreshold in limits.items():
1693  if not math.isnan(maskThreshold):
1694  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1695  isrFunctions.makeThresholdMask(
1696  maskedImage=dataView,
1697  threshold=maskThreshold,
1698  growFootprints=0,
1699  maskName=maskName
1700  )
1701 
1702  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1703  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1704  afwImage.PARENT)
1705  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1706  self.config.suspectMaskName])
1707  if numpy.all(maskView.getArray() & maskVal > 0):
1708  badAmp = True
1709  maskView |= maskView.getPlaneBitMask("BAD")
1710 
1711  return badAmp
1712 
1713  def overscanCorrection(self, ccdExposure, amp):
1714  """Apply overscan correction in place.
1715 
1716  This method does initial pixel rejection of the overscan
1717  region. The overscan can also be optionally segmented to
1718  allow for discontinuous overscan responses to be fit
1719  separately. The actual overscan subtraction is performed by
1720  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1721  which is called here after the amplifier is preprocessed.
1722 
1723  Parameters
1724  ----------
1725  ccdExposure : `lsst.afw.image.Exposure`
1726  Exposure to have overscan correction performed.
1727  amp : `lsst.afw.table.AmpInfoCatalog`
1728  The amplifier to consider while correcting the overscan.
1729 
1730  Returns
1731  -------
1732  overscanResults : `lsst.pipe.base.Struct`
1733  Result struct with components:
1734  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1735  Value or fit subtracted from the amplifier image data.
1736  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1737  Value or fit subtracted from the overscan image data.
1738  - ``overscanImage`` : `lsst.afw.image.Image`
1739  Image of the overscan region with the overscan
1740  correction applied. This quantity is used to estimate
1741  the amplifier read noise empirically.
1742 
1743  Raises
1744  ------
1745  RuntimeError
1746  Raised if the ``amp`` does not contain raw pixel information.
1747 
1748  See Also
1749  --------
1750  lsst.ip.isr.isrFunctions.overscanCorrection
1751  """
1752  if not amp.getHasRawInfo():
1753  raise RuntimeError("This method must be executed on an amp with raw information.")
1754 
1755  if amp.getRawHorizontalOverscanBBox().isEmpty():
1756  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1757  return None
1758 
1759  statControl = afwMath.StatisticsControl()
1760  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1761 
1762  # Determine the bounding boxes
1763  dataBBox = amp.getRawDataBBox()
1764  oscanBBox = amp.getRawHorizontalOverscanBBox()
1765  dx0 = 0
1766  dx1 = 0
1767 
1768  prescanBBox = amp.getRawPrescanBBox()
1769  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1770  dx0 += self.config.overscanNumLeadingColumnsToSkip
1771  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1772  else:
1773  dx0 += self.config.overscanNumTrailingColumnsToSkip
1774  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1775 
1776  # Determine if we need to work on subregions of the amplifier and overscan.
1777  imageBBoxes = []
1778  overscanBBoxes = []
1779 
1780  if ((self.config.overscanBiasJump and
1781  self.config.overscanBiasJumpLocation) and
1782  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1783  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1784  self.config.overscanBiasJumpDevices)):
1785  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1786  yLower = self.config.overscanBiasJumpLocation
1787  yUpper = dataBBox.getHeight() - yLower
1788  else:
1789  yUpper = self.config.overscanBiasJumpLocation
1790  yLower = dataBBox.getHeight() - yUpper
1791 
1792  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1793  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1794  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1795  lsst.geom.Extent2I(dx0, 0),
1796  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1797  yLower)))
1798 
1799  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1800  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1801  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1802  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1803  yUpper)))
1804  else:
1805  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1806  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1807  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1808  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1809  oscanBBox.getHeight())))
1810 
1811  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1812  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1813  ampImage = ccdExposure.maskedImage[imageBBox]
1814  overscanImage = ccdExposure.maskedImage[overscanBBox]
1815 
1816  overscanArray = overscanImage.image.array
1817  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1818  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1819  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1820 
1821  statControl = afwMath.StatisticsControl()
1822  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1823 
1824  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1825  overscanImage=overscanImage,
1826  fitType=self.config.overscanFitType,
1827  order=self.config.overscanOrder,
1828  collapseRej=self.config.overscanNumSigmaClip,
1829  statControl=statControl,
1830  overscanIsInt=self.config.overscanIsInt
1831  )
1832 
1833  # Measure average overscan levels and record them in the metadata.
1834  levelStat = afwMath.MEDIAN
1835  sigmaStat = afwMath.STDEVCLIP
1836 
1837  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1838  self.config.qa.flatness.nIter)
1839  metadata = ccdExposure.getMetadata()
1840  ampNum = amp.getName()
1841  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1842  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1843  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1844  else:
1845  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1846  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1847  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1848 
1849  return overscanResults
1850 
1851  def updateVariance(self, ampExposure, amp, overscanImage=None):
1852  """Set the variance plane using the amplifier gain and read noise
1853 
1854  The read noise is calculated from the ``overscanImage`` if the
1855  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1856  the value from the amplifier data is used.
1857 
1858  Parameters
1859  ----------
1860  ampExposure : `lsst.afw.image.Exposure`
1861  Exposure to process.
1862  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1863  Amplifier detector data.
1864  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1865  Image of overscan, required only for empirical read noise.
1866 
1867  See also
1868  --------
1869  lsst.ip.isr.isrFunctions.updateVariance
1870  """
1871  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1872  gain = amp.getGain()
1873 
1874  if math.isnan(gain):
1875  gain = 1.0
1876  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1877  elif gain <= 0:
1878  patchedGain = 1.0
1879  self.log.warn("Gain for amp %s == %g <= 0; setting to %f." %
1880  (amp.getName(), gain, patchedGain))
1881  gain = patchedGain
1882 
1883  if self.config.doEmpiricalReadNoise and overscanImage is None:
1884  self.log.info("Overscan is none for EmpiricalReadNoise.")
1885 
1886  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1887  stats = afwMath.StatisticsControl()
1888  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1889  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1890  self.log.info("Calculated empirical read noise for amp %s: %f.", amp.getName(), readNoise)
1891  else:
1892  readNoise = amp.getReadNoise()
1893 
1894  isrFunctions.updateVariance(
1895  maskedImage=ampExposure.getMaskedImage(),
1896  gain=gain,
1897  readNoise=readNoise,
1898  )
1899 
1900  def darkCorrection(self, exposure, darkExposure, invert=False):
1901  """!Apply dark correction in place.
1902 
1903  Parameters
1904  ----------
1905  exposure : `lsst.afw.image.Exposure`
1906  Exposure to process.
1907  darkExposure : `lsst.afw.image.Exposure`
1908  Dark exposure of the same size as ``exposure``.
1909  invert : `Bool`, optional
1910  If True, re-add the dark to an already corrected image.
1911 
1912  Raises
1913  ------
1914  RuntimeError
1915  Raised if either ``exposure`` or ``darkExposure`` do not
1916  have their dark time defined.
1917 
1918  See Also
1919  --------
1920  lsst.ip.isr.isrFunctions.darkCorrection
1921  """
1922  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1923  if math.isnan(expScale):
1924  raise RuntimeError("Exposure darktime is NAN.")
1925  if darkExposure.getInfo().getVisitInfo() is not None:
1926  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1927  else:
1928  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1929  # so getDarkTime() does not exist.
1930  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1931  darkScale = 1.0
1932 
1933  if math.isnan(darkScale):
1934  raise RuntimeError("Dark calib darktime is NAN.")
1935  isrFunctions.darkCorrection(
1936  maskedImage=exposure.getMaskedImage(),
1937  darkMaskedImage=darkExposure.getMaskedImage(),
1938  expScale=expScale,
1939  darkScale=darkScale,
1940  invert=invert,
1941  trimToFit=self.config.doTrimToMatchCalib
1942  )
1943 
1944  def doLinearize(self, detector):
1945  """!Check if linearization is needed for the detector cameraGeom.
1946 
1947  Checks config.doLinearize and the linearity type of the first
1948  amplifier.
1949 
1950  Parameters
1951  ----------
1952  detector : `lsst.afw.cameraGeom.Detector`
1953  Detector to get linearity type from.
1954 
1955  Returns
1956  -------
1957  doLinearize : `Bool`
1958  If True, linearization should be performed.
1959  """
1960  return self.config.doLinearize and \
1961  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1962 
1963  def flatCorrection(self, exposure, flatExposure, invert=False):
1964  """!Apply flat correction in place.
1965 
1966  Parameters
1967  ----------
1968  exposure : `lsst.afw.image.Exposure`
1969  Exposure to process.
1970  flatExposure : `lsst.afw.image.Exposure`
1971  Flat exposure of the same size as ``exposure``.
1972  invert : `Bool`, optional
1973  If True, unflatten an already flattened image.
1974 
1975  See Also
1976  --------
1977  lsst.ip.isr.isrFunctions.flatCorrection
1978  """
1979  isrFunctions.flatCorrection(
1980  maskedImage=exposure.getMaskedImage(),
1981  flatMaskedImage=flatExposure.getMaskedImage(),
1982  scalingType=self.config.flatScalingType,
1983  userScale=self.config.flatUserScale,
1984  invert=invert,
1985  trimToFit=self.config.doTrimToMatchCalib
1986  )
1987 
1988  def saturationDetection(self, exposure, amp):
1989  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1990 
1991  Parameters
1992  ----------
1993  exposure : `lsst.afw.image.Exposure`
1994  Exposure to process. Only the amplifier DataSec is processed.
1995  amp : `lsst.afw.table.AmpInfoCatalog`
1996  Amplifier detector data.
1997 
1998  See Also
1999  --------
2000  lsst.ip.isr.isrFunctions.makeThresholdMask
2001  """
2002  if not math.isnan(amp.getSaturation()):
2003  maskedImage = exposure.getMaskedImage()
2004  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2005  isrFunctions.makeThresholdMask(
2006  maskedImage=dataView,
2007  threshold=amp.getSaturation(),
2008  growFootprints=0,
2009  maskName=self.config.saturatedMaskName,
2010  )
2011 
2012  def saturationInterpolation(self, exposure):
2013  """!Interpolate over saturated pixels, in place.
2014 
2015  This method should be called after `saturationDetection`, to
2016  ensure that the saturated pixels have been identified in the
2017  SAT mask. It should also be called after `assembleCcd`, since
2018  saturated regions may cross amplifier boundaries.
2019 
2020  Parameters
2021  ----------
2022  exposure : `lsst.afw.image.Exposure`
2023  Exposure to process.
2024 
2025  See Also
2026  --------
2027  lsst.ip.isr.isrTask.saturationDetection
2028  lsst.ip.isr.isrFunctions.interpolateFromMask
2029  """
2030  isrFunctions.interpolateFromMask(
2031  maskedImage=exposure.getMaskedImage(),
2032  fwhm=self.config.fwhm,
2033  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2034  maskNameList=list(self.config.saturatedMaskName),
2035  )
2036 
2037  def suspectDetection(self, exposure, amp):
2038  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2039 
2040  Parameters
2041  ----------
2042  exposure : `lsst.afw.image.Exposure`
2043  Exposure to process. Only the amplifier DataSec is processed.
2044  amp : `lsst.afw.table.AmpInfoCatalog`
2045  Amplifier detector data.
2046 
2047  See Also
2048  --------
2049  lsst.ip.isr.isrFunctions.makeThresholdMask
2050 
2051  Notes
2052  -----
2053  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2054  This is intended to indicate pixels that may be affected by unknown systematics;
2055  for example if non-linearity corrections above a certain level are unstable
2056  then that would be a useful value for suspectLevel. A value of `nan` indicates
2057  that no such level exists and no pixels are to be masked as suspicious.
2058  """
2059  suspectLevel = amp.getSuspectLevel()
2060  if math.isnan(suspectLevel):
2061  return
2062 
2063  maskedImage = exposure.getMaskedImage()
2064  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2065  isrFunctions.makeThresholdMask(
2066  maskedImage=dataView,
2067  threshold=suspectLevel,
2068  growFootprints=0,
2069  maskName=self.config.suspectMaskName,
2070  )
2071 
2072  def maskDefect(self, exposure, defectBaseList):
2073  """!Mask defects using mask plane "BAD", in place.
2074 
2075  Parameters
2076  ----------
2077  exposure : `lsst.afw.image.Exposure`
2078  Exposure to process.
2079  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2080  `lsst.afw.image.DefectBase`.
2081  List of defects to mask and interpolate.
2082 
2083  Notes
2084  -----
2085  Call this after CCD assembly, since defects may cross amplifier boundaries.
2086  """
2087  maskedImage = exposure.getMaskedImage()
2088  if not isinstance(defectBaseList, Defects):
2089  # Promotes DefectBase to Defect
2090  defectList = Defects(defectBaseList)
2091  else:
2092  defectList = defectBaseList
2093  defectList.maskPixels(maskedImage, maskName="BAD")
2094 
2095  if self.config.numEdgeSuspect > 0:
2096  goodBBox = maskedImage.getBBox()
2097  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2098  goodBBox.grow(-self.config.numEdgeSuspect)
2099  # Mask pixels outside goodBBox as SUSPECT
2100  SourceDetectionTask.setEdgeBits(
2101  maskedImage,
2102  goodBBox,
2103  maskedImage.getMask().getPlaneBitMask("SUSPECT")
2104  )
2105 
2106  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2107  """Mask and interpolate defects using mask plane "BAD", in place.
2108 
2109  Parameters
2110  ----------
2111  exposure : `lsst.afw.image.Exposure`
2112  Exposure to process.
2113  defectBaseList : `List` of `Defects`
2114 
2115  """
2116  self.maskDefects(exposure, defectBaseList)
2117  isrFunctions.interpolateFromMask(
2118  maskedImage=exposure.getMaskedImage(),
2119  fwhm=self.config.fwhm,
2120  growSaturatedFootprints=0,
2121  maskNameList=["BAD"],
2122  )
2123 
2124  def maskNan(self, exposure):
2125  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2126 
2127  Parameters
2128  ----------
2129  exposure : `lsst.afw.image.Exposure`
2130  Exposure to process.
2131 
2132  Notes
2133  -----
2134  We mask over all NaNs, including those that are masked with
2135  other bits (because those may or may not be interpolated over
2136  later, and we want to remove all NaNs). Despite this
2137  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2138  the historical name.
2139  """
2140  maskedImage = exposure.getMaskedImage()
2141 
2142  # Find and mask NaNs
2143  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2144  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2145  numNans = maskNans(maskedImage, maskVal)
2146  self.metadata.set("NUMNANS", numNans)
2147  if numNans > 0:
2148  self.log.warn(f"There were {numNans} unmasked NaNs.")
2149 
2150  def maskAndInterpolateNan(self, exposure):
2151  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2152 
2153  Parameters
2154  ----------
2155  exposure : `lsst.afw.image.Exposure`
2156  Exposure to process.
2157 
2158  See Also
2159  --------
2160  lsst.ip.isr.isrTask.maskNan()
2161  """
2162  self.maskNan(exposure)
2163  isrFunctions.interpolateFromMask(
2164  maskedImage=exposure.getMaskedImage(),
2165  fwhm=self.config.fwhm,
2166  growSaturatedFootprints=0,
2167  maskNameList=["UNMASKEDNAN"],
2168  )
2169 
2170  def measureBackground(self, exposure, IsrQaConfig=None):
2171  """Measure the image background in subgrids, for quality control purposes.
2172 
2173  Parameters
2174  ----------
2175  exposure : `lsst.afw.image.Exposure`
2176  Exposure to process.
2177  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2178  Configuration object containing parameters on which background
2179  statistics and subgrids to use.
2180  """
2181  if IsrQaConfig is not None:
2182  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2183  IsrQaConfig.flatness.nIter)
2184  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2185  statsControl.setAndMask(maskVal)
2186  maskedImage = exposure.getMaskedImage()
2187  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2188  skyLevel = stats.getValue(afwMath.MEDIAN)
2189  skySigma = stats.getValue(afwMath.STDEVCLIP)
2190  self.log.info("Flattened sky level: %f +/- %f." % (skyLevel, skySigma))
2191  metadata = exposure.getMetadata()
2192  metadata.set('SKYLEVEL', skyLevel)
2193  metadata.set('SKYSIGMA', skySigma)
2194 
2195  # calcluating flatlevel over the subgrids
2196  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2197  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2198  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2199  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2200  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2201  skyLevels = numpy.zeros((nX, nY))
2202 
2203  for j in range(nY):
2204  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2205  for i in range(nX):
2206  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2207 
2208  xLLC = xc - meshXHalf
2209  yLLC = yc - meshYHalf
2210  xURC = xc + meshXHalf - 1
2211  yURC = yc + meshYHalf - 1
2212 
2213  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2214  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2215 
2216  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2217 
2218  good = numpy.where(numpy.isfinite(skyLevels))
2219  skyMedian = numpy.median(skyLevels[good])
2220  flatness = (skyLevels[good] - skyMedian) / skyMedian
2221  flatness_rms = numpy.std(flatness)
2222  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2223 
2224  self.log.info("Measuring sky levels in %dx%d grids: %f." % (nX, nY, skyMedian))
2225  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f." %
2226  (nX, nY, flatness_pp, flatness_rms))
2227 
2228  metadata.set('FLATNESS_PP', float(flatness_pp))
2229  metadata.set('FLATNESS_RMS', float(flatness_rms))
2230  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2231  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2232  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2233 
2234  def roughZeroPoint(self, exposure):
2235  """Set an approximate magnitude zero point for the exposure.
2236 
2237  Parameters
2238  ----------
2239  exposure : `lsst.afw.image.Exposure`
2240  Exposure to process.
2241  """
2242  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2243  if filterName in self.config.fluxMag0T1:
2244  fluxMag0 = self.config.fluxMag0T1[filterName]
2245  else:
2246  self.log.warn("No rough magnitude zero point set for filter %s." % filterName)
2247  fluxMag0 = self.config.defaultFluxMag0T1
2248 
2249  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2250  if not expTime > 0: # handle NaN as well as <= 0
2251  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2252  return
2253 
2254  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2255  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2256 
2257  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2258  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2259 
2260  Parameters
2261  ----------
2262  ccdExposure : `lsst.afw.image.Exposure`
2263  Exposure to process.
2264  fpPolygon : `lsst.afw.geom.Polygon`
2265  Polygon in focal plane coordinates.
2266  """
2267  # Get ccd corners in focal plane coordinates
2268  ccd = ccdExposure.getDetector()
2269  fpCorners = ccd.getCorners(FOCAL_PLANE)
2270  ccdPolygon = Polygon(fpCorners)
2271 
2272  # Get intersection of ccd corners with fpPolygon
2273  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2274 
2275  # Transform back to pixel positions and build new polygon
2276  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2277  validPolygon = Polygon(ccdPoints)
2278  ccdExposure.getInfo().setValidPolygon(validPolygon)
2279 
2280  @contextmanager
2281  def flatContext(self, exp, flat, dark=None):
2282  """Context manager that applies and removes flats and darks,
2283  if the task is configured to apply them.
2284 
2285  Parameters
2286  ----------
2287  exp : `lsst.afw.image.Exposure`
2288  Exposure to process.
2289  flat : `lsst.afw.image.Exposure`
2290  Flat exposure the same size as ``exp``.
2291  dark : `lsst.afw.image.Exposure`, optional
2292  Dark exposure the same size as ``exp``.
2293 
2294  Yields
2295  ------
2296  exp : `lsst.afw.image.Exposure`
2297  The flat and dark corrected exposure.
2298  """
2299  if self.config.doDark and dark is not None:
2300  self.darkCorrection(exp, dark)
2301  if self.config.doFlat:
2302  self.flatCorrection(exp, flat)
2303  try:
2304  yield exp
2305  finally:
2306  if self.config.doFlat:
2307  self.flatCorrection(exp, flat, invert=True)
2308  if self.config.doDark and dark is not None:
2309  self.darkCorrection(exp, dark, invert=True)
2310 
2311  def debugView(self, exposure, stepname):
2312  """Utility function to examine ISR exposure at different stages.
2313 
2314  Parameters
2315  ----------
2316  exposure : `lsst.afw.image.Exposure`
2317  Exposure to view.
2318  stepname : `str`
2319  State of processing to view.
2320  """
2321  frame = getDebugFrame(self._display, stepname)
2322  if frame:
2323  display = getDisplay(frame)
2324  display.scale('asinh', 'zscale')
2325  display.mtv(exposure)
2326  prompt = "Press Enter to continue [c]... "
2327  while True:
2328  ans = input(prompt).lower()
2329  if ans in ("", "c",):
2330  break
2331 
2332 
2333 class FakeAmp(object):
2334  """A Detector-like object that supports returning gain and saturation level
2335 
2336  This is used when the input exposure does not have a detector.
2337 
2338  Parameters
2339  ----------
2340  exposure : `lsst.afw.image.Exposure`
2341  Exposure to generate a fake amplifier for.
2342  config : `lsst.ip.isr.isrTaskConfig`
2343  Configuration to apply to the fake amplifier.
2344  """
2345 
2346  def __init__(self, exposure, config):
2347  self._bbox = exposure.getBBox(afwImage.LOCAL)
2349  self._gain = config.gain
2350  self._readNoise = config.readNoise
2351  self._saturation = config.saturation
2352 
2353  def getBBox(self):
2354  return self._bbox
2355 
2356  def getRawBBox(self):
2357  return self._bbox
2358 
2359  def getHasRawInfo(self):
2360  return True # but see getRawHorizontalOverscanBBox()
2361 
2363  return self._RawHorizontalOverscanBBox
2364 
2365  def getGain(self):
2366  return self._gain
2367 
2368  def getReadNoise(self):
2369  return self._readNoise
2370 
2371  def getSaturation(self):
2372  return self._saturation
2373 
2374  def getSuspectLevel(self):
2375  return float("NaN")
2376 
2377 
2378 class RunIsrConfig(pexConfig.Config):
2379  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2380 
2381 
2382 class RunIsrTask(pipeBase.CmdLineTask):
2383  """Task to wrap the default IsrTask to allow it to be retargeted.
2384 
2385  The standard IsrTask can be called directly from a command line
2386  program, but doing so removes the ability of the task to be
2387  retargeted. As most cameras override some set of the IsrTask
2388  methods, this would remove those data-specific methods in the
2389  output post-ISR images. This wrapping class fixes the issue,
2390  allowing identical post-ISR images to be generated by both the
2391  processCcd and isrTask code.
2392  """
2393  ConfigClass = RunIsrConfig
2394  _DefaultName = "runIsr"
2395 
2396  def __init__(self, *args, **kwargs):
2397  super().__init__(*args, **kwargs)
2398  self.makeSubtask("isr")
2399 
2400  def runDataRef(self, dataRef):
2401  """
2402  Parameters
2403  ----------
2404  dataRef : `lsst.daf.persistence.ButlerDataRef`
2405  data reference of the detector data to be processed
2406 
2407  Returns
2408  -------
2409  result : `pipeBase.Struct`
2410  Result struct with component:
2411 
2412  - exposure : `lsst.afw.image.Exposure`
2413  Post-ISR processed exposure.
2414  """
2415  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:769
def runDataRef(self, sensorRef)
Definition: isrTask.py:1463
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2170
def debugView(self, exposure, stepname)
Definition: isrTask.py:2311
def __init__(self, kwargs)
Definition: isrTask.py:759
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1558
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:878
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:838
def runDataRef(self, dataRef)
Definition: isrTask.py:2400
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1026
def __init__(self, args, kwargs)
Definition: isrTask.py:2396
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:821
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2234
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2106
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2362
def maskNan(self, exposure)
Definition: isrTask.py:2124
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:808
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2072
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1713
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1605
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1963
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:875
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1517
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1900
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1944
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2257
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1642
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:831
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2281
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1851
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2150
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2037
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2012
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1988
def __init__(self, exposure, config)
Definition: isrTask.py:2346