lsst.ip.isr  17.0.1-24-g44c3907
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["instrument"],
149  )
150  illumMaskedImage = pipeBase.InputDatasetField(
151  doc="Input illumination correction.",
152  name="illum",
153  scalar=True,
154  storageClass="MaskedImageF",
155  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
156  )
157 
158  # output datasets
159  outputExposure = pipeBase.OutputDatasetField(
160  doc="Output ISR processed exposure.",
161  name="postISRCCD",
162  scalar=True,
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = pipeBase.OutputDatasetField(
167  doc="Output Overscan-subtracted thumbnail image.",
168  name="OssThumb",
169  scalar=True,
170  storageClass="Thumbnail",
171  dimensions=["instrument", "visit", "detector"],
172  )
173  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
174  doc="Output flat-corrected thumbnail image.",
175  name="FlattenedThumb",
176  scalar=True,
177  storageClass="TextStorage",
178  dimensions=["instrument", "visit", "detector"],
179  )
180 
181  quantum = pipeBase.QuantumConfig(
182  dimensions=["visit", "detector", "instrument"],
183  )
184 
185 
186  datasetType = pexConfig.Field(
187  dtype=str,
188  doc="Dataset type for input data; users will typically leave this alone, "
189  "but camera-specific ISR tasks will override it",
190  default="raw",
191  )
192 
193  fallbackFilterName = pexConfig.Field(
194  dtype=str,
195  doc="Fallback default filter name for calibrations.",
196  optional=True
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
202  )
203  fwhm = pexConfig.Field(
204  dtype=float,
205  doc="FWHM of PSF in arcseconds.",
206  default=1.0,
207  )
208  qa = pexConfig.ConfigField(
209  dtype=isrQa.IsrQaConfig,
210  doc="QA related configuration options.",
211  )
212 
213  # Image conversion configuration
214  doConvertIntToFloat = pexConfig.Field(
215  dtype=bool,
216  doc="Convert integer raw images to floating point values?",
217  default=True,
218  )
219 
220  # Saturated pixel handling.
221  doSaturation = pexConfig.Field(
222  dtype=bool,
223  doc="Mask saturated pixels? NB: this is totally independent of the"
224  " interpolation option - this is ONLY setting the bits in the mask."
225  " To have them interpolated make sure doSaturationInterpolation=True",
226  default=True,
227  )
228  saturatedMaskName = pexConfig.Field(
229  dtype=str,
230  doc="Name of mask plane to use in saturation detection and interpolation",
231  default="SAT",
232  )
233  saturation = pexConfig.Field(
234  dtype=float,
235  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
236  default=float("NaN"),
237  )
238  growSaturationFootprintSize = pexConfig.Field(
239  dtype=int,
240  doc="Number of pixels by which to grow the saturation footprints",
241  default=1,
242  )
243 
244  # Suspect pixel handling.
245  doSuspect = pexConfig.Field(
246  dtype=bool,
247  doc="Mask suspect pixels?",
248  default=True,
249  )
250  suspectMaskName = pexConfig.Field(
251  dtype=str,
252  doc="Name of mask plane to use for suspect pixels",
253  default="SUSPECT",
254  )
255  numEdgeSuspect = pexConfig.Field(
256  dtype=int,
257  doc="Number of edge pixels to be flagged as untrustworthy.",
258  default=0,
259  )
260 
261  # Initial masking options.
262  doSetBadRegions = pexConfig.Field(
263  dtype=bool,
264  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
265  default=True,
266  )
267  badStatistic = pexConfig.ChoiceField(
268  dtype=str,
269  doc="How to estimate the average value for BAD regions.",
270  default='MEANCLIP',
271  allowed={
272  "MEANCLIP": "Correct using the (clipped) mean of good data",
273  "MEDIAN": "Correct using the median of the good data",
274  },
275  )
276 
277  # Overscan subtraction configuration.
278  doOverscan = pexConfig.Field(
279  dtype=bool,
280  doc="Do overscan subtraction?",
281  default=True,
282  )
283  overscanFitType = pexConfig.ChoiceField(
284  dtype=str,
285  doc="The method for fitting the overscan bias level.",
286  default='MEDIAN',
287  allowed={
288  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
289  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
290  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
291  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
292  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
293  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
294  "MEAN": "Correct using the mean of the overscan region",
295  "MEANCLIP": "Correct using a clipped mean of the overscan region",
296  "MEDIAN": "Correct using the median of the overscan region",
297  },
298  )
299  overscanOrder = pexConfig.Field(
300  dtype=int,
301  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
302  "or number of spline knots if overscan fit type is a spline."),
303  default=1,
304  )
305  overscanNumSigmaClip = pexConfig.Field(
306  dtype=float,
307  doc="Rejection threshold (sigma) for collapsing overscan before fit",
308  default=3.0,
309  )
310  overscanIsInt = pexConfig.Field(
311  dtype=bool,
312  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
313  default=True,
314  )
315  overscanNumLeadingColumnsToSkip = pexConfig.Field(
316  dtype=int,
317  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
318  default=0,
319  )
320  overscanNumTrailingColumnsToSkip = pexConfig.Field(
321  dtype=int,
322  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
323  default=0,
324  )
325  overscanMaxDev = pexConfig.Field(
326  dtype=float,
327  doc="Maximum deviation from the median for overscan",
328  default=1000.0, check=lambda x: x > 0
329  )
330  overscanBiasJump = pexConfig.Field(
331  dtype=bool,
332  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
333  default=False,
334  )
335  overscanBiasJumpKeyword = pexConfig.Field(
336  dtype=str,
337  doc="Header keyword containing information about devices.",
338  default="NO_SUCH_KEY",
339  )
340  overscanBiasJumpDevices = pexConfig.ListField(
341  dtype=str,
342  doc="List of devices that need piecewise overscan correction.",
343  default=(),
344  )
345  overscanBiasJumpLocation = pexConfig.Field(
346  dtype=int,
347  doc="Location of bias jump along y-axis.",
348  default=0,
349  )
350 
351  # Amplifier to CCD assembly configuration
352  doAssembleCcd = pexConfig.Field(
353  dtype=bool,
354  default=True,
355  doc="Assemble amp-level exposures into a ccd-level exposure?"
356  )
357  assembleCcd = pexConfig.ConfigurableField(
358  target=AssembleCcdTask,
359  doc="CCD assembly task",
360  )
361 
362  # General calibration configuration.
363  doAssembleIsrExposures = pexConfig.Field(
364  dtype=bool,
365  default=False,
366  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
367  )
368  doTrimToMatchCalib = pexConfig.Field(
369  dtype=bool,
370  default=False,
371  doc="Trim raw data to match calibration bounding boxes?"
372  )
373 
374  # Bias subtraction.
375  doBias = pexConfig.Field(
376  dtype=bool,
377  doc="Apply bias frame correction?",
378  default=True,
379  )
380  biasDataProductName = pexConfig.Field(
381  dtype=str,
382  doc="Name of the bias data product",
383  default="bias",
384  )
385 
386  # Variance construction
387  doVariance = pexConfig.Field(
388  dtype=bool,
389  doc="Calculate variance?",
390  default=True
391  )
392  gain = pexConfig.Field(
393  dtype=float,
394  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
395  default=float("NaN"),
396  )
397  readNoise = pexConfig.Field(
398  dtype=float,
399  doc="The read noise to use if no Detector is present in the Exposure",
400  default=0.0,
401  )
402  doEmpiricalReadNoise = pexConfig.Field(
403  dtype=bool,
404  default=False,
405  doc="Calculate empirical read noise instead of value from AmpInfo data?"
406  )
407 
408  # Linearization.
409  doLinearize = pexConfig.Field(
410  dtype=bool,
411  doc="Correct for nonlinearity of the detector's response?",
412  default=True,
413  )
414 
415  # Crosstalk.
416  doCrosstalk = pexConfig.Field(
417  dtype=bool,
418  doc="Apply intra-CCD crosstalk correction?",
419  default=False,
420  )
421  doCrosstalkBeforeAssemble = pexConfig.Field(
422  dtype=bool,
423  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
424  default=True,
425  )
426  crosstalk = pexConfig.ConfigurableField(
427  target=CrosstalkTask,
428  doc="Intra-CCD crosstalk correction",
429  )
430 
431  # Masking options.
432  doDefect = pexConfig.Field(
433  dtype=bool,
434  doc="Apply correction for CCD defects, e.g. hot pixels?",
435  default=True,
436  )
437  numEdgeSuspect = pexConfig.Field(
438  dtype=int,
439  doc="Number of edge pixels to be flagged as untrustworthy.",
440  default=0,
441  )
442  doNanMasking = pexConfig.Field(
443  dtype=bool,
444  doc="Mask NAN pixels?",
445  default=True,
446  )
447  doWidenSaturationTrails = pexConfig.Field(
448  dtype=bool,
449  doc="Widen bleed trails based on their width?",
450  default=True
451  )
452 
453  # Brighter-Fatter correction.
454  doBrighterFatter = pexConfig.Field(
455  dtype=bool,
456  default=False,
457  doc="Apply the brighter fatter correction"
458  )
459  brighterFatterLevel = pexConfig.ChoiceField(
460  dtype=str,
461  default="DETECTOR",
462  doc="The level at which to correct for brighter-fatter.",
463  allowed={
464  "AMP": "Every amplifier treated separately.",
465  "DETECTOR": "One kernel per detector",
466  }
467  )
468  brighterFatterKernelFile = pexConfig.Field(
469  dtype=str,
470  default='',
471  doc="Kernel file used for the brighter fatter correction"
472  )
473  brighterFatterMaxIter = pexConfig.Field(
474  dtype=int,
475  default=10,
476  doc="Maximum number of iterations for the brighter fatter correction"
477  )
478  brighterFatterThreshold = pexConfig.Field(
479  dtype=float,
480  default=1000,
481  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
482  " absolute value of the difference between the current corrected image and the one"
483  " from the previous iteration summed over all the pixels."
484  )
485  brighterFatterApplyGain = pexConfig.Field(
486  dtype=bool,
487  default=True,
488  doc="Should the gain be applied when applying the brighter fatter correction?"
489  )
490 
491  # Dark subtraction.
492  doDark = pexConfig.Field(
493  dtype=bool,
494  doc="Apply dark frame correction?",
495  default=True,
496  )
497  darkDataProductName = pexConfig.Field(
498  dtype=str,
499  doc="Name of the dark data product",
500  default="dark",
501  )
502 
503  # Camera-specific stray light removal.
504  doStrayLight = pexConfig.Field(
505  dtype=bool,
506  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
507  default=False,
508  )
509  strayLight = pexConfig.ConfigurableField(
510  target=StrayLightTask,
511  doc="y-band stray light correction"
512  )
513 
514  # Flat correction.
515  doFlat = pexConfig.Field(
516  dtype=bool,
517  doc="Apply flat field correction?",
518  default=True,
519  )
520  flatDataProductName = pexConfig.Field(
521  dtype=str,
522  doc="Name of the flat data product",
523  default="flat",
524  )
525  flatScalingType = pexConfig.ChoiceField(
526  dtype=str,
527  doc="The method for scaling the flat on the fly.",
528  default='USER',
529  allowed={
530  "USER": "Scale by flatUserScale",
531  "MEAN": "Scale by the inverse of the mean",
532  "MEDIAN": "Scale by the inverse of the median",
533  },
534  )
535  flatUserScale = pexConfig.Field(
536  dtype=float,
537  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
538  default=1.0,
539  )
540  doTweakFlat = pexConfig.Field(
541  dtype=bool,
542  doc="Tweak flats to match observed amplifier ratios?",
543  default=False
544  )
545 
546  # Amplifier normalization based on gains instead of using flats configuration.
547  doApplyGains = pexConfig.Field(
548  dtype=bool,
549  doc="Correct the amplifiers for their gains instead of applying flat correction",
550  default=False,
551  )
552  normalizeGains = pexConfig.Field(
553  dtype=bool,
554  doc="Normalize all the amplifiers in each CCD to have the same median value.",
555  default=False,
556  )
557 
558  # Fringe correction.
559  doFringe = pexConfig.Field(
560  dtype=bool,
561  doc="Apply fringe correction?",
562  default=True,
563  )
564  fringe = pexConfig.ConfigurableField(
565  target=FringeTask,
566  doc="Fringe subtraction task",
567  )
568  fringeAfterFlat = pexConfig.Field(
569  dtype=bool,
570  doc="Do fringe subtraction after flat-fielding?",
571  default=True,
572  )
573 
574  # Distortion model application.
575  doAddDistortionModel = pexConfig.Field(
576  dtype=bool,
577  doc="Apply a distortion model based on camera geometry to the WCS?",
578  default=True,
579  )
580 
581  # Initial CCD-level background statistics options.
582  doMeasureBackground = pexConfig.Field(
583  dtype=bool,
584  doc="Measure the background level on the reduced image?",
585  default=False,
586  )
587 
588  # Camera-specific masking configuration.
589  doCameraSpecificMasking = pexConfig.Field(
590  dtype=bool,
591  doc="Mask camera-specific bad regions?",
592  default=False,
593  )
594  masking = pexConfig.ConfigurableField(
595  target=MaskingTask,
596  doc="Masking task."
597  )
598 
599  # Interpolation options.
600 
601  doInterpolate = pexConfig.Field(
602  dtype=bool,
603  doc="Interpolate masked pixels?",
604  default=True,
605  )
606  doSaturationInterpolation = pexConfig.Field(
607  dtype=bool,
608  doc="Perform interpolation over pixels masked as saturated?"
609  " NB: This is independent of doSaturation; if that is False this plane"
610  " will likely be blank, resulting in a no-op here.",
611  default=True,
612  )
613  doNanInterpolation = pexConfig.Field(
614  dtype=bool,
615  doc="Perform interpolation over pixels masked as NaN?"
616  " NB: This is independent of doNanMasking; if that is False this plane"
617  " will likely be blank, resulting in a no-op here.",
618  default=True,
619  )
620  doNanInterpAfterFlat = pexConfig.Field(
621  dtype=bool,
622  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
623  "also have to interpolate them before flat-fielding."),
624  default=False,
625  )
626  maskListToInterpolate = pexConfig.ListField(
627  dtype=str,
628  doc="List of mask planes that should be interpolated.",
629  default=['SAT', 'BAD', 'UNMASKEDNAN'],
630  )
631  doSaveInterpPixels = pexConfig.Field(
632  dtype=bool,
633  doc="Save a copy of the pre-interpolated pixel values?",
634  default=False,
635  )
636 
637  # Default photometric calibration options.
638  fluxMag0T1 = pexConfig.DictField(
639  keytype=str,
640  itemtype=float,
641  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
642  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
643  ))
644  )
645  defaultFluxMag0T1 = pexConfig.Field(
646  dtype=float,
647  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
648  default=pow(10.0, 0.4*28.0)
649  )
650 
651  # Vignette correction configuration.
652  doVignette = pexConfig.Field(
653  dtype=bool,
654  doc="Apply vignetting parameters?",
655  default=False,
656  )
657  vignette = pexConfig.ConfigurableField(
658  target=VignetteTask,
659  doc="Vignetting task.",
660  )
661 
662  # Transmission curve configuration.
663  doAttachTransmissionCurve = pexConfig.Field(
664  dtype=bool,
665  default=False,
666  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
667  )
668  doUseOpticsTransmission = pexConfig.Field(
669  dtype=bool,
670  default=True,
671  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
672  )
673  doUseFilterTransmission = pexConfig.Field(
674  dtype=bool,
675  default=True,
676  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
677  )
678  doUseSensorTransmission = pexConfig.Field(
679  dtype=bool,
680  default=True,
681  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
682  )
683  doUseAtmosphereTransmission = pexConfig.Field(
684  dtype=bool,
685  default=True,
686  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
687  )
688 
689  # Illumination correction.
690  doIlluminationCorrection = pexConfig.Field(
691  dtype=bool,
692  default=False,
693  doc="Perform illumination correction?"
694  )
695  illuminationCorrectionDataProductName = pexConfig.Field(
696  dtype=str,
697  doc="Name of the illumination correction data product.",
698  default="illumcor",
699  )
700  illumScale = pexConfig.Field(
701  dtype=float,
702  doc="Scale factor for the illumination correction.",
703  default=1.0,
704  )
705  illumFilters = pexConfig.ListField(
706  dtype=str,
707  default=[],
708  doc="Only perform illumination correction for these filters."
709  )
710 
711  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
712  doWrite = pexConfig.Field(
713  dtype=bool,
714  doc="Persist postISRCCD?",
715  default=True,
716  )
717 
718  def validate(self):
719  super().validate()
720  if self.doFlat and self.doApplyGains:
721  raise ValueError("You may not specify both doFlat and doApplyGains")
722  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
723  self.config.maskListToInterpolate.append("SAT")
724  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
725  self.config.maskListToInterpolate.append("UNMASKEDNAN")
726 
727 
728 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
729  """Apply common instrument signature correction algorithms to a raw frame.
730 
731  The process for correcting imaging data is very similar from
732  camera to camera. This task provides a vanilla implementation of
733  doing these corrections, including the ability to turn certain
734  corrections off if they are not needed. The inputs to the primary
735  method, `run()`, are a raw exposure to be corrected and the
736  calibration data products. The raw input is a single chip sized
737  mosaic of all amps including overscans and other non-science
738  pixels. The method `runDataRef()` identifies and defines the
739  calibration data products, and is intended for use by a
740  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
741  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
742  subclassed for different camera, although the most camera specific
743  methods have been split into subtasks that can be redirected
744  appropriately.
745 
746  The __init__ method sets up the subtasks for ISR processing, using
747  the defaults from `lsst.ip.isr`.
748 
749  Parameters
750  ----------
751  args : `list`
752  Positional arguments passed to the Task constructor. None used at this time.
753  kwargs : `dict`, optional
754  Keyword arguments passed on to the Task constructor. None used at this time.
755  """
756  ConfigClass = IsrTaskConfig
757  _DefaultName = "isr"
758 
759  def __init__(self, **kwargs):
760  super().__init__(**kwargs)
761  self.makeSubtask("assembleCcd")
762  self.makeSubtask("crosstalk")
763  self.makeSubtask("strayLight")
764  self.makeSubtask("fringe")
765  self.makeSubtask("masking")
766  self.makeSubtask("vignette")
767 
768  @classmethod
769  def getInputDatasetTypes(cls, config):
770  inputTypeDict = super().getInputDatasetTypes(config)
771 
772  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
773  # need because the configuration tells us we will not be bothering with the
774  # correction that uses that IDT.
775  if config.doBias is not True:
776  inputTypeDict.pop("bias", None)
777  if config.doLinearize is not True:
778  inputTypeDict.pop("linearizer", None)
779  if config.doCrosstalk is not True:
780  inputTypeDict.pop("crosstalkSources", None)
781  if config.doBrighterFatter is not True:
782  inputTypeDict.pop("bfKernel", None)
783  if config.doDefect is not True:
784  inputTypeDict.pop("defects", None)
785  if config.doDark is not True:
786  inputTypeDict.pop("dark", None)
787  if config.doFlat is not True:
788  inputTypeDict.pop("flat", None)
789  if config.doAttachTransmissionCurve is not True:
790  inputTypeDict.pop("opticsTransmission", None)
791  inputTypeDict.pop("filterTransmission", None)
792  inputTypeDict.pop("sensorTransmission", None)
793  inputTypeDict.pop("atmosphereTransmission", None)
794  if config.doUseOpticsTransmission is not True:
795  inputTypeDict.pop("opticsTransmission", None)
796  if config.doUseFilterTransmission is not True:
797  inputTypeDict.pop("filterTransmission", None)
798  if config.doUseSensorTransmission is not True:
799  inputTypeDict.pop("sensorTransmission", None)
800  if config.doUseAtmosphereTransmission is not True:
801  inputTypeDict.pop("atmosphereTransmission", None)
802  if config.doIlluminationCorrection is not True:
803  inputTypeDict.pop("illumMaskedImage", None)
804 
805  return inputTypeDict
806 
807  @classmethod
808  def getOutputDatasetTypes(cls, config):
809  outputTypeDict = super().getOutputDatasetTypes(config)
810 
811  if config.qa.doThumbnailOss is not True:
812  outputTypeDict.pop("outputOssThumbnail", None)
813  if config.qa.doThumbnailFlattened is not True:
814  outputTypeDict.pop("outputFlattenedThumbnail", None)
815  if config.doWrite is not True:
816  outputTypeDict.pop("outputExposure", None)
817 
818  return outputTypeDict
819 
820  @classmethod
821  def getPrerequisiteDatasetTypes(cls, config):
822  # Input calibration datasets should not constrain the QuantumGraph
823  # (it'd be confusing if not having flats just silently resulted in no
824  # data being processed). Our nomenclature for that is that these are
825  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
826  names = set(cls.getInputDatasetTypes(config))
827  names.remove("ccdExposure")
828  return names
829 
830  @classmethod
831  def getPerDatasetTypeDimensions(cls, config):
832  # Input calibration datasets of different types (i.e. flat, bias) need
833  # not have the same validity range. That makes calibration_label
834  # (which maps directly to a validity range) a "per-DatasetType
835  # dimension".
836  return frozenset(["calibration_label"])
837 
838  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
839  try:
840  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
841  except Exception as e:
842  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
843 
844  inputData['isGen3'] = True
845 
846  if self.config.doLinearize is True:
847  if 'linearizer' not in inputData.keys():
848  detector = inputData['camera'][inputData['detectorNum']]
849  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
850  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
851 
852  if inputData['defects'] is not None:
853  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
854  # masking expects a list of defects defined by their bounding box
855  if not isinstance(inputData["defects"], Defects):
856  inputData["defects"] = Defects.fromTable(inputData["defects"])
857 
858  # Broken: DM-17169
859  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
860  # fixed for non-HSC cameras in the future.
861  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
862  # if self.config.doCrosstalk else None)
863 
864  # Broken: DM-17152
865  # Fringes are not tested to be handled correctly by Gen3 butler.
866  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
867  # assembler=self.assembleCcd
868  # if self.config.doAssembleIsrExposures else None)
869  # if self.config.doFringe and
870  # self.fringe.checkFilter(inputData['ccdExposure'])
871  # else pipeBase.Struct(fringes=None))
872 
873  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
874 
875  def makeDatasetType(self, dsConfig):
876  return super().makeDatasetType(dsConfig)
877 
878  def readIsrData(self, dataRef, rawExposure):
879  """!Retrieve necessary frames for instrument signature removal.
880 
881  Pre-fetching all required ISR data products limits the IO
882  required by the ISR. Any conflict between the calibration data
883  available and that needed for ISR is also detected prior to
884  doing processing, allowing it to fail quickly.
885 
886  Parameters
887  ----------
888  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
889  Butler reference of the detector data to be processed
890  rawExposure : `afw.image.Exposure`
891  The raw exposure that will later be corrected with the
892  retrieved calibration data; should not be modified in this
893  method.
894 
895  Returns
896  -------
897  result : `lsst.pipe.base.Struct`
898  Result struct with components (which may be `None`):
899  - ``bias``: bias calibration frame (`afw.image.Exposure`)
900  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
901  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
902  - ``dark``: dark calibration frame (`afw.image.Exposure`)
903  - ``flat``: flat calibration frame (`afw.image.Exposure`)
904  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
905  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
906  - ``fringes``: `lsst.pipe.base.Struct` with components:
907  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
908  - ``seed``: random seed derived from the ccdExposureId for random
909  number generator (`uint32`).
910  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
911  A ``TransmissionCurve`` that represents the throughput of the optics,
912  to be evaluated in focal-plane coordinates.
913  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
914  A ``TransmissionCurve`` that represents the throughput of the filter
915  itself, to be evaluated in focal-plane coordinates.
916  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
917  A ``TransmissionCurve`` that represents the throughput of the sensor
918  itself, to be evaluated in post-assembly trimmed detector coordinates.
919  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
920  A ``TransmissionCurve`` that represents the throughput of the
921  atmosphere, assumed to be spatially constant.
922  - ``strayLightData`` : `object`
923  An opaque object containing calibration information for
924  stray-light correction. If `None`, no correction will be
925  performed.
926  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
927 
928  Raises
929  ------
930  NotImplementedError :
931  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
932  """
933  ccd = rawExposure.getDetector()
934  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
935  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
936  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
937  if self.config.doBias else None)
938  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
939  linearizer = (dataRef.get("linearizer", immediate=True)
940  if self.doLinearize(ccd) else None)
941  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
942  if self.config.doCrosstalk else None)
943  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
944  if self.config.doDark else None)
945  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
946  if self.config.doFlat else None)
947 
948  brighterFatterKernel = None
949  if self.config.doBrighterFatter is True:
950 
951  # Use the new-style cp_pipe version of the kernel is it exists.
952  try:
953  brighterFatterKernel = dataRef.get("brighterFatterKernel")
954  except NoResults:
955  # Fall back to the old-style numpy-ndarray style kernel if necessary.
956  try:
957  brighterFatterKernel = dataRef.get("bfKernel")
958  except NoResults:
959  brighterFatterKernel = None
960  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
961  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
962  # this detector, or raise an error.
963  if self.config.brighterFatterLevel == 'DETECTOR':
964  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
965  else:
966  # TODO DM-15631 for implementing this
967  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
968 
969  defectList = (dataRef.get("defects")
970  if self.config.doDefect else None)
971  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
972  if self.config.doAssembleIsrExposures else None)
973  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
974  else pipeBase.Struct(fringes=None))
975 
976  if self.config.doAttachTransmissionCurve:
977  opticsTransmission = (dataRef.get("transmission_optics")
978  if self.config.doUseOpticsTransmission else None)
979  filterTransmission = (dataRef.get("transmission_filter")
980  if self.config.doUseFilterTransmission else None)
981  sensorTransmission = (dataRef.get("transmission_sensor")
982  if self.config.doUseSensorTransmission else None)
983  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
984  if self.config.doUseAtmosphereTransmission else None)
985  else:
986  opticsTransmission = None
987  filterTransmission = None
988  sensorTransmission = None
989  atmosphereTransmission = None
990 
991  if self.config.doStrayLight:
992  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
993  else:
994  strayLightData = None
995 
996  illumMaskedImage = (self.getIsrExposure(dataRef,
997  self.config.illuminationCorrectionDataProductName).getMaskedImage()
998  if (self.config.doIlluminationCorrection and
999  filterName in self.config.illumFilters)
1000  else None)
1001 
1002  # Struct should include only kwargs to run()
1003  return pipeBase.Struct(bias=biasExposure,
1004  linearizer=linearizer,
1005  crosstalkSources=crosstalkSources,
1006  dark=darkExposure,
1007  flat=flatExposure,
1008  bfKernel=brighterFatterKernel,
1009  defects=defectList,
1010  fringes=fringeStruct,
1011  opticsTransmission=opticsTransmission,
1012  filterTransmission=filterTransmission,
1013  sensorTransmission=sensorTransmission,
1014  atmosphereTransmission=atmosphereTransmission,
1015  strayLightData=strayLightData,
1016  illumMaskedImage=illumMaskedImage
1017  )
1018 
1019  @pipeBase.timeMethod
1020  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1021  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1022  opticsTransmission=None, filterTransmission=None,
1023  sensorTransmission=None, atmosphereTransmission=None,
1024  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1025  isGen3=False,
1026  ):
1027  """!Perform instrument signature removal on an exposure.
1028 
1029  Steps included in the ISR processing, in order performed, are:
1030  - saturation and suspect pixel masking
1031  - overscan subtraction
1032  - CCD assembly of individual amplifiers
1033  - bias subtraction
1034  - variance image construction
1035  - linearization of non-linear response
1036  - crosstalk masking
1037  - brighter-fatter correction
1038  - dark subtraction
1039  - fringe correction
1040  - stray light subtraction
1041  - flat correction
1042  - masking of known defects and camera specific features
1043  - vignette calculation
1044  - appending transmission curve and distortion model
1045 
1046  Parameters
1047  ----------
1048  ccdExposure : `lsst.afw.image.Exposure`
1049  The raw exposure that is to be run through ISR. The
1050  exposure is modified by this method.
1051  camera : `lsst.afw.cameraGeom.Camera`, optional
1052  The camera geometry for this exposure. Used to select the
1053  distortion model appropriate for this data.
1054  bias : `lsst.afw.image.Exposure`, optional
1055  Bias calibration frame.
1056  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1057  Functor for linearization.
1058  crosstalkSources : `list`, optional
1059  List of possible crosstalk sources.
1060  dark : `lsst.afw.image.Exposure`, optional
1061  Dark calibration frame.
1062  flat : `lsst.afw.image.Exposure`, optional
1063  Flat calibration frame.
1064  bfKernel : `numpy.ndarray`, optional
1065  Brighter-fatter kernel.
1066  defects : `lsst.meas.algorithms.Defects`, optional
1067  List of defects.
1068  fringes : `lsst.pipe.base.Struct`, optional
1069  Struct containing the fringe correction data, with
1070  elements:
1071  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1072  - ``seed``: random seed derived from the ccdExposureId for random
1073  number generator (`uint32`)
1074  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1075  A ``TransmissionCurve`` that represents the throughput of the optics,
1076  to be evaluated in focal-plane coordinates.
1077  filterTransmission : `lsst.afw.image.TransmissionCurve`
1078  A ``TransmissionCurve`` that represents the throughput of the filter
1079  itself, to be evaluated in focal-plane coordinates.
1080  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1081  A ``TransmissionCurve`` that represents the throughput of the sensor
1082  itself, to be evaluated in post-assembly trimmed detector coordinates.
1083  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1084  A ``TransmissionCurve`` that represents the throughput of the
1085  atmosphere, assumed to be spatially constant.
1086  detectorNum : `int`, optional
1087  The integer number for the detector to process.
1088  isGen3 : bool, optional
1089  Flag this call to run() as using the Gen3 butler environment.
1090  strayLightData : `object`, optional
1091  Opaque object containing calibration information for stray-light
1092  correction. If `None`, no correction will be performed.
1093  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1094  Illumination correction image.
1095 
1096  Returns
1097  -------
1098  result : `lsst.pipe.base.Struct`
1099  Result struct with component:
1100  - ``exposure`` : `afw.image.Exposure`
1101  The fully ISR corrected exposure.
1102  - ``outputExposure`` : `afw.image.Exposure`
1103  An alias for `exposure`
1104  - ``ossThumb`` : `numpy.ndarray`
1105  Thumbnail image of the exposure after overscan subtraction.
1106  - ``flattenedThumb`` : `numpy.ndarray`
1107  Thumbnail image of the exposure after flat-field correction.
1108 
1109  Raises
1110  ------
1111  RuntimeError
1112  Raised if a configuration option is set to True, but the
1113  required calibration data has not been specified.
1114 
1115  Notes
1116  -----
1117  The current processed exposure can be viewed by setting the
1118  appropriate lsstDebug entries in the `debug.display`
1119  dictionary. The names of these entries correspond to some of
1120  the IsrTaskConfig Boolean options, with the value denoting the
1121  frame to use. The exposure is shown inside the matching
1122  option check and after the processing of that step has
1123  finished. The steps with debug points are:
1124 
1125  doAssembleCcd
1126  doBias
1127  doCrosstalk
1128  doBrighterFatter
1129  doDark
1130  doFringe
1131  doStrayLight
1132  doFlat
1133 
1134  In addition, setting the "postISRCCD" entry displays the
1135  exposure after all ISR processing has finished.
1136 
1137  """
1138 
1139  if isGen3 is True:
1140  # Gen3 currently cannot automatically do configuration overrides.
1141  # DM-15257 looks to discuss this issue.
1142 
1143  self.config.doFringe = False
1144 
1145  # Configure input exposures;
1146  if detectorNum is None:
1147  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1148 
1149  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1150  bias = self.ensureExposure(bias, camera, detectorNum)
1151  dark = self.ensureExposure(dark, camera, detectorNum)
1152  flat = self.ensureExposure(flat, camera, detectorNum)
1153  else:
1154  if isinstance(ccdExposure, ButlerDataRef):
1155  return self.runDataRef(ccdExposure)
1156 
1157  ccd = ccdExposure.getDetector()
1158  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1159 
1160  if not ccd:
1161  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1162  ccd = [FakeAmp(ccdExposure, self.config)]
1163 
1164  # Validate Input
1165  if self.config.doBias and bias is None:
1166  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1167  if self.doLinearize(ccd) and linearizer is None:
1168  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1169  if self.config.doBrighterFatter and bfKernel is None:
1170  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1171  if self.config.doDark and dark is None:
1172  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1173  if self.config.doFlat and flat is None:
1174  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1175  if self.config.doDefect and defects is None:
1176  raise RuntimeError("Must supply defects if config.doDefect=True.")
1177  if self.config.doAddDistortionModel and camera is None:
1178  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1179  if (self.config.doFringe and filterName in self.fringe.config.filters and
1180  fringes.fringes is None):
1181  # The `fringes` object needs to be a pipeBase.Struct, as
1182  # we use it as a `dict` for the parameters of
1183  # `FringeTask.run()`. The `fringes.fringes` `list` may
1184  # not be `None` if `doFringe=True`. Otherwise, raise.
1185  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1186  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1187  illumMaskedImage is None):
1188  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1189 
1190  # Begin ISR processing.
1191  if self.config.doConvertIntToFloat:
1192  self.log.info("Converting exposure to floating point values.")
1193  ccdExposure = self.convertIntToFloat(ccdExposure)
1194 
1195  # Amplifier level processing.
1196  overscans = []
1197  for amp in ccd:
1198  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1199  if ccdExposure.getBBox().contains(amp.getBBox()):
1200  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1201  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1202 
1203  if self.config.doOverscan and not badAmp:
1204  # Overscan correction on amp-by-amp basis.
1205  overscanResults = self.overscanCorrection(ccdExposure, amp)
1206  self.log.debug(f"Corrected overscan for amplifier {amp.getName()}.")
1207  if self.config.qa is not None and self.config.qa.saveStats is True:
1208  if isinstance(overscanResults.overscanFit, float):
1209  qaMedian = overscanResults.overscanFit
1210  qaStdev = float("NaN")
1211  else:
1212  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1213  afwMath.MEDIAN | afwMath.STDEVCLIP)
1214  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1215  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1216 
1217  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1218  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1219  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1220  (amp.getName(), qaMedian, qaStdev))
1221  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1222  else:
1223  if badAmp:
1224  self.log.warn(f"Amplifier {amp.getName()} is bad.")
1225  overscanResults = None
1226 
1227  overscans.append(overscanResults if overscanResults is not None else None)
1228  else:
1229  self.log.info(f"Skipped OSCAN for {amp.getName()}.")
1230 
1231  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1232  self.log.info("Applying crosstalk correction.")
1233  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1234  self.debugView(ccdExposure, "doCrosstalk")
1235 
1236  if self.config.doAssembleCcd:
1237  self.log.info("Assembling CCD from amplifiers.")
1238  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1239 
1240  if self.config.expectWcs and not ccdExposure.getWcs():
1241  self.log.warn("No WCS found in input exposure.")
1242  self.debugView(ccdExposure, "doAssembleCcd")
1243 
1244  ossThumb = None
1245  if self.config.qa.doThumbnailOss:
1246  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1247 
1248  if self.config.doBias:
1249  self.log.info("Applying bias correction.")
1250  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1251  trimToFit=self.config.doTrimToMatchCalib)
1252  self.debugView(ccdExposure, "doBias")
1253 
1254  if self.config.doVariance:
1255  for amp, overscanResults in zip(ccd, overscans):
1256  if ccdExposure.getBBox().contains(amp.getBBox()):
1257  self.log.debug(f"Constructing variance map for amplifer {amp.getName()}.")
1258  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1259  if overscanResults is not None:
1260  self.updateVariance(ampExposure, amp,
1261  overscanImage=overscanResults.overscanImage)
1262  else:
1263  self.updateVariance(ampExposure, amp,
1264  overscanImage=None)
1265  if self.config.qa is not None and self.config.qa.saveStats is True:
1266  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1267  afwMath.MEDIAN | afwMath.STDEVCLIP)
1268  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1269  qaStats.getValue(afwMath.MEDIAN))
1270  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1271  qaStats.getValue(afwMath.STDEVCLIP))
1272  self.log.debug(" Variance stats for amplifer %s: %f +/- %f." %
1273  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1274  qaStats.getValue(afwMath.STDEVCLIP)))
1275 
1276  if self.doLinearize(ccd):
1277  self.log.info("Applying linearizer.")
1278  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1279 
1280  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1281  self.log.info("Applying crosstalk correction.")
1282  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1283  self.debugView(ccdExposure, "doCrosstalk")
1284 
1285  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1286  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1287  if self.config.doDefect:
1288  self.log.info("Masking defects.")
1289  self.maskDefect(ccdExposure, defects)
1290 
1291  if self.config.doNanMasking:
1292  self.log.info("Masking NAN value pixels.")
1293  self.maskNan(ccdExposure)
1294 
1295  if self.config.doWidenSaturationTrails:
1296  self.log.info("Widening saturation trails.")
1297  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1298 
1299  if self.config.doCameraSpecificMasking:
1300  self.log.info("Masking regions for camera specific reasons.")
1301  self.masking.run(ccdExposure)
1302 
1303  if self.config.doBrighterFatter:
1304  # We need to apply flats and darks before we can interpolate, and we
1305  # need to interpolate before we do B-F, but we do B-F without the
1306  # flats and darks applied so we can work in units of electrons or holes.
1307  # This context manager applies and then removes the darks and flats.
1308  #
1309  # We also do not want to interpolate values here, so operate on temporary
1310  # images so we can apply only the BF-correction and roll back the
1311  # interpolation.
1312  interpExp = ccdExposure.clone()
1313  with self.flatContext(interpExp, flat, dark):
1314  isrFunctions.interpolateFromMask(
1315  maskedImage=interpExp.getMaskedImage(),
1316  fwhm=self.config.fwhm,
1317  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1318  maskNameList=self.config.maskListToInterpolate
1319  )
1320  bfExp = interpExp.clone()
1321 
1322  self.log.info("Applying brighter fatter correction.")
1323  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1324  self.config.brighterFatterMaxIter,
1325  self.config.brighterFatterThreshold,
1326  self.config.brighterFatterApplyGain
1327  )
1328  if bfResults[1] == self.config.brighterFatterMaxIter:
1329  self.log.warn("Brighter fatter correction did not converge, final difference {bfResults[0]}.")
1330  else:
1331  self.log.info("Finished brighter fatter correction in {bfResults[1]} iterations.")
1332  image = ccdExposure.getMaskedImage().getImage()
1333  bfCorr = bfExp.getMaskedImage().getImage()
1334  bfCorr -= interpExp.getMaskedImage().getImage()
1335  image += bfCorr
1336 
1337  self.debugView(ccdExposure, "doBrighterFatter")
1338 
1339  if self.config.doDark:
1340  self.log.info("Applying dark correction.")
1341  self.darkCorrection(ccdExposure, dark)
1342  self.debugView(ccdExposure, "doDark")
1343 
1344  if self.config.doFringe and not self.config.fringeAfterFlat:
1345  self.log.info("Applying fringe correction before flat.")
1346  self.fringe.run(ccdExposure, **fringes.getDict())
1347  self.debugView(ccdExposure, "doFringe")
1348 
1349  if self.config.doStrayLight:
1350  if strayLightData is not None:
1351  self.log.info("Applying stray light correction.")
1352  self.strayLight.run(ccdExposure, strayLightData)
1353  self.debugView(ccdExposure, "doStrayLight")
1354  else:
1355  self.log.debug("Skipping stray light correction: no data found for this image.")
1356 
1357  if self.config.doFlat:
1358  self.log.info("Applying flat correction.")
1359  self.flatCorrection(ccdExposure, flat)
1360  self.debugView(ccdExposure, "doFlat")
1361 
1362  if self.config.doApplyGains:
1363  self.log.info("Applying gain correction instead of flat.")
1364  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1365 
1366  if self.config.doFringe and self.config.fringeAfterFlat:
1367  self.log.info("Applying fringe correction after flat.")
1368  self.fringe.run(ccdExposure, **fringes.getDict())
1369 
1370  if self.config.doVignette:
1371  self.log.info("Constructing Vignette polygon.")
1372  self.vignettePolygon = self.vignette.run(ccdExposure)
1373 
1374  if self.config.vignette.doWriteVignettePolygon:
1375  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1376 
1377  if self.config.doAttachTransmissionCurve:
1378  self.log.info("Adding transmission curves.")
1379  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1380  filterTransmission=filterTransmission,
1381  sensorTransmission=sensorTransmission,
1382  atmosphereTransmission=atmosphereTransmission)
1383 
1384  if self.config.doAddDistortionModel:
1385  self.log.info("Adding a distortion model to the WCS.")
1386  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1387 
1388  flattenedThumb = None
1389  if self.config.qa.doThumbnailFlattened:
1390  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1391 
1392  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1393  self.log.info("Performing illumination correction.")
1394  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1395  illumMaskedImage, illumScale=self.config.illumScale,
1396  trimToFit=self.config.doTrimToMatchCalib)
1397 
1398  preInterpExp = None
1399  if self.config.doSaveInterpPixels:
1400  preInterpExp = ccdExposure.clone()
1401 
1402  # Reset and interpolate bad pixels.
1403  #
1404  # Large contiguous bad regions (which should have the BAD mask
1405  # bit set) should have their values set to the image median.
1406  # This group should include defects and bad amplifiers. As the
1407  # area covered by these defects are large, there's little
1408  # reason to expect that interpolation would provide a more
1409  # useful value.
1410  #
1411  # Smaller defects can be safely interpolated after the larger
1412  # regions have had their pixel values reset. This ensures
1413  # that the remaining defects adjacent to bad amplifiers (as an
1414  # example) do not attempt to interpolate extreme values.
1415  if self.config.doSetBadRegions:
1416  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1417  if badPixelCount > 0:
1418  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1419 
1420  if self.config.doInterpolate:
1421  self.log.info("Interpolating masked pixels.")
1422  isrFunctions.interpolateFromMask(
1423  maskedImage=ccdExposure.getMaskedImage(),
1424  fwhm=self.config.fwhm,
1425  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1426  maskNameList=list(self.config.maskListToInterpolate)
1427  )
1428 
1429  self.roughZeroPoint(ccdExposure)
1430 
1431  if self.config.doMeasureBackground:
1432  self.log.info("Measuring background level.")
1433  self.measureBackground(ccdExposure, self.config.qa)
1434 
1435  if self.config.qa is not None and self.config.qa.saveStats is True:
1436  for amp in ccd:
1437  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1438  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1439  afwMath.MEDIAN | afwMath.STDEVCLIP)
1440  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1441  qaStats.getValue(afwMath.MEDIAN))
1442  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1443  qaStats.getValue(afwMath.STDEVCLIP))
1444  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1445  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1446  qaStats.getValue(afwMath.STDEVCLIP)))
1447 
1448  self.debugView(ccdExposure, "postISRCCD")
1449 
1450  return pipeBase.Struct(
1451  exposure=ccdExposure,
1452  ossThumb=ossThumb,
1453  flattenedThumb=flattenedThumb,
1454 
1455  preInterpolatedExposure=preInterpExp,
1456  outputExposure=ccdExposure,
1457  outputOssThumbnail=ossThumb,
1458  outputFlattenedThumbnail=flattenedThumb,
1459  )
1460 
1461  @pipeBase.timeMethod
1462  def runDataRef(self, sensorRef):
1463  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1464 
1465  This method contains the `CmdLineTask` interface to the ISR
1466  processing. All IO is handled here, freeing the `run()` method
1467  to manage only pixel-level calculations. The steps performed
1468  are:
1469  - Read in necessary detrending/isr/calibration data.
1470  - Process raw exposure in `run()`.
1471  - Persist the ISR-corrected exposure as "postISRCCD" if
1472  config.doWrite=True.
1473 
1474  Parameters
1475  ----------
1476  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1477  DataRef of the detector data to be processed
1478 
1479  Returns
1480  -------
1481  result : `lsst.pipe.base.Struct`
1482  Result struct with component:
1483  - ``exposure`` : `afw.image.Exposure`
1484  The fully ISR corrected exposure.
1485 
1486  Raises
1487  ------
1488  RuntimeError
1489  Raised if a configuration option is set to True, but the
1490  required calibration data does not exist.
1491 
1492  """
1493  self.log.info("Performing ISR on sensor %s." % (sensorRef.dataId))
1494 
1495  ccdExposure = sensorRef.get(self.config.datasetType)
1496 
1497  camera = sensorRef.get("camera")
1498  if camera is None and self.config.doAddDistortionModel:
1499  raise RuntimeError("config.doAddDistortionModel is True "
1500  "but could not get a camera from the butler.")
1501  isrData = self.readIsrData(sensorRef, ccdExposure)
1502 
1503  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1504 
1505  if self.config.doWrite:
1506  sensorRef.put(result.exposure, "postISRCCD")
1507  if result.preInterpolatedExposure is not None:
1508  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1509  if result.ossThumb is not None:
1510  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1511  if result.flattenedThumb is not None:
1512  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1513 
1514  return result
1515 
1516  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1517  """!Retrieve a calibration dataset for removing instrument signature.
1518 
1519  Parameters
1520  ----------
1521 
1522  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1523  DataRef of the detector data to find calibration datasets
1524  for.
1525  datasetType : `str`
1526  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1527  immediate : `Bool`
1528  If True, disable butler proxies to enable error handling
1529  within this routine.
1530 
1531  Returns
1532  -------
1533  exposure : `lsst.afw.image.Exposure`
1534  Requested calibration frame.
1535 
1536  Raises
1537  ------
1538  RuntimeError
1539  Raised if no matching calibration frame can be found.
1540  """
1541  try:
1542  exp = dataRef.get(datasetType, immediate=immediate)
1543  except Exception as exc1:
1544  if not self.config.fallbackFilterName:
1545  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1546  try:
1547  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1548  except Exception as exc2:
1549  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1550  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1551  self.log.warn("Using fallback calibration from filter %s." % self.config.fallbackFilterName)
1552 
1553  if self.config.doAssembleIsrExposures:
1554  exp = self.assembleCcd.assembleCcd(exp)
1555  return exp
1556 
1557  def ensureExposure(self, inputExp, camera, detectorNum):
1558  """Ensure that the data returned by Butler is a fully constructed exposure.
1559 
1560  ISR requires exposure-level image data for historical reasons, so if we did
1561  not recieve that from Butler, construct it from what we have, modifying the
1562  input in place.
1563 
1564  Parameters
1565  ----------
1566  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1567  `lsst.afw.image.ImageF`
1568  The input data structure obtained from Butler.
1569  camera : `lsst.afw.cameraGeom.camera`
1570  The camera associated with the image. Used to find the appropriate
1571  detector.
1572  detectorNum : `int`
1573  The detector this exposure should match.
1574 
1575  Returns
1576  -------
1577  inputExp : `lsst.afw.image.Exposure`
1578  The re-constructed exposure, with appropriate detector parameters.
1579 
1580  Raises
1581  ------
1582  TypeError
1583  Raised if the input data cannot be used to construct an exposure.
1584  """
1585  if isinstance(inputExp, afwImage.DecoratedImageU):
1586  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1587  elif isinstance(inputExp, afwImage.ImageF):
1588  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1589  elif isinstance(inputExp, afwImage.MaskedImageF):
1590  inputExp = afwImage.makeExposure(inputExp)
1591  elif isinstance(inputExp, afwImage.Exposure):
1592  pass
1593  elif inputExp is None:
1594  # Assume this will be caught by the setup if it is a problem.
1595  return inputExp
1596  else:
1597  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}.")
1598 
1599  if inputExp.getDetector() is None:
1600  inputExp.setDetector(camera[detectorNum])
1601 
1602  return inputExp
1603 
1604  def convertIntToFloat(self, exposure):
1605  """Convert exposure image from uint16 to float.
1606 
1607  If the exposure does not need to be converted, the input is
1608  immediately returned. For exposures that are converted to use
1609  floating point pixels, the variance is set to unity and the
1610  mask to zero.
1611 
1612  Parameters
1613  ----------
1614  exposure : `lsst.afw.image.Exposure`
1615  The raw exposure to be converted.
1616 
1617  Returns
1618  -------
1619  newexposure : `lsst.afw.image.Exposure`
1620  The input ``exposure``, converted to floating point pixels.
1621 
1622  Raises
1623  ------
1624  RuntimeError
1625  Raised if the exposure type cannot be converted to float.
1626 
1627  """
1628  if isinstance(exposure, afwImage.ExposureF):
1629  # Nothing to be done
1630  self.log.debug("Exposure already of type float.")
1631  return exposure
1632  if not hasattr(exposure, "convertF"):
1633  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1634 
1635  newexposure = exposure.convertF()
1636  newexposure.variance[:] = 1
1637  newexposure.mask[:] = 0x0
1638 
1639  return newexposure
1640 
1641  def maskAmplifier(self, ccdExposure, amp, defects):
1642  """Identify bad amplifiers, saturated and suspect pixels.
1643 
1644  Parameters
1645  ----------
1646  ccdExposure : `lsst.afw.image.Exposure`
1647  Input exposure to be masked.
1648  amp : `lsst.afw.table.AmpInfoCatalog`
1649  Catalog of parameters defining the amplifier on this
1650  exposure to mask.
1651  defects : `lsst.meas.algorithms.Defects`
1652  List of defects. Used to determine if the entire
1653  amplifier is bad.
1654 
1655  Returns
1656  -------
1657  badAmp : `Bool`
1658  If this is true, the entire amplifier area is covered by
1659  defects and unusable.
1660 
1661  """
1662  maskedImage = ccdExposure.getMaskedImage()
1663 
1664  badAmp = False
1665 
1666  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1667  # comparison with current defects definition.
1668  if defects is not None:
1669  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1670 
1671  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1672  # association with pixels in current ccdExposure).
1673  if badAmp:
1674  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1675  afwImage.PARENT)
1676  maskView = dataView.getMask()
1677  maskView |= maskView.getPlaneBitMask("BAD")
1678  del maskView
1679  return badAmp
1680 
1681  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1682  # Saturation and suspect pixels can be masked now, though.
1683  limits = dict()
1684  if self.config.doSaturation and not badAmp:
1685  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1686  if self.config.doSuspect and not badAmp:
1687  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1688  if math.isfinite(self.config.saturation):
1689  limits.update({self.config.saturatedMaskName: self.config.saturation})
1690 
1691  for maskName, maskThreshold in limits.items():
1692  if not math.isnan(maskThreshold):
1693  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1694  isrFunctions.makeThresholdMask(
1695  maskedImage=dataView,
1696  threshold=maskThreshold,
1697  growFootprints=0,
1698  maskName=maskName
1699  )
1700 
1701  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1702  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1703  afwImage.PARENT)
1704  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1705  self.config.suspectMaskName])
1706  if numpy.all(maskView.getArray() & maskVal > 0):
1707  badAmp = True
1708  maskView |= maskView.getPlaneBitMask("BAD")
1709 
1710  return badAmp
1711 
1712  def overscanCorrection(self, ccdExposure, amp):
1713  """Apply overscan correction in place.
1714 
1715  This method does initial pixel rejection of the overscan
1716  region. The overscan can also be optionally segmented to
1717  allow for discontinuous overscan responses to be fit
1718  separately. The actual overscan subtraction is performed by
1719  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1720  which is called here after the amplifier is preprocessed.
1721 
1722  Parameters
1723  ----------
1724  ccdExposure : `lsst.afw.image.Exposure`
1725  Exposure to have overscan correction performed.
1726  amp : `lsst.afw.table.AmpInfoCatalog`
1727  The amplifier to consider while correcting the overscan.
1728 
1729  Returns
1730  -------
1731  overscanResults : `lsst.pipe.base.Struct`
1732  Result struct with components:
1733  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1734  Value or fit subtracted from the amplifier image data.
1735  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1736  Value or fit subtracted from the overscan image data.
1737  - ``overscanImage`` : `lsst.afw.image.Image`
1738  Image of the overscan region with the overscan
1739  correction applied. This quantity is used to estimate
1740  the amplifier read noise empirically.
1741 
1742  Raises
1743  ------
1744  RuntimeError
1745  Raised if the ``amp`` does not contain raw pixel information.
1746 
1747  See Also
1748  --------
1749  lsst.ip.isr.isrFunctions.overscanCorrection
1750  """
1751  if not amp.getHasRawInfo():
1752  raise RuntimeError("This method must be executed on an amp with raw information.")
1753 
1754  if amp.getRawHorizontalOverscanBBox().isEmpty():
1755  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1756  return None
1757 
1758  statControl = afwMath.StatisticsControl()
1759  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1760 
1761  # Determine the bounding boxes
1762  dataBBox = amp.getRawDataBBox()
1763  oscanBBox = amp.getRawHorizontalOverscanBBox()
1764  dx0 = 0
1765  dx1 = 0
1766 
1767  prescanBBox = amp.getRawPrescanBBox()
1768  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1769  dx0 += self.config.overscanNumLeadingColumnsToSkip
1770  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1771  else:
1772  dx0 += self.config.overscanNumTrailingColumnsToSkip
1773  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1774 
1775  # Determine if we need to work on subregions of the amplifier and overscan.
1776  imageBBoxes = []
1777  overscanBBoxes = []
1778 
1779  if ((self.config.overscanBiasJump and
1780  self.config.overscanBiasJumpLocation) and
1781  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1782  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1783  self.config.overscanBiasJumpDevices)):
1784  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1785  yLower = self.config.overscanBiasJumpLocation
1786  yUpper = dataBBox.getHeight() - yLower
1787  else:
1788  yUpper = self.config.overscanBiasJumpLocation
1789  yLower = dataBBox.getHeight() - yUpper
1790 
1791  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1792  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1793  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1794  lsst.geom.Extent2I(dx0, 0),
1795  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1796  yLower)))
1797 
1798  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1799  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1800  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1801  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1802  yUpper)))
1803  else:
1804  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1805  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1806  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1807  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1808  oscanBBox.getHeight())))
1809 
1810  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1811  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1812  ampImage = ccdExposure.maskedImage[imageBBox]
1813  overscanImage = ccdExposure.maskedImage[overscanBBox]
1814 
1815  overscanArray = overscanImage.image.array
1816  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1817  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1818  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1819 
1820  statControl = afwMath.StatisticsControl()
1821  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1822 
1823  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1824  overscanImage=overscanImage,
1825  fitType=self.config.overscanFitType,
1826  order=self.config.overscanOrder,
1827  collapseRej=self.config.overscanNumSigmaClip,
1828  statControl=statControl,
1829  overscanIsInt=self.config.overscanIsInt
1830  )
1831 
1832  # Measure average overscan levels and record them in the metadata.
1833  levelStat = afwMath.MEDIAN
1834  sigmaStat = afwMath.STDEVCLIP
1835 
1836  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1837  self.config.qa.flatness.nIter)
1838  metadata = ccdExposure.getMetadata()
1839  ampNum = amp.getName()
1840  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1841  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1842  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1843  else:
1844  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1845  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1846  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1847 
1848  return overscanResults
1849 
1850  def updateVariance(self, ampExposure, amp, overscanImage=None):
1851  """Set the variance plane using the amplifier gain and read noise
1852 
1853  The read noise is calculated from the ``overscanImage`` if the
1854  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1855  the value from the amplifier data is used.
1856 
1857  Parameters
1858  ----------
1859  ampExposure : `lsst.afw.image.Exposure`
1860  Exposure to process.
1861  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1862  Amplifier detector data.
1863  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1864  Image of overscan, required only for empirical read noise.
1865 
1866  See also
1867  --------
1868  lsst.ip.isr.isrFunctions.updateVariance
1869  """
1870  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1871  gain = amp.getGain()
1872 
1873  if math.isnan(gain):
1874  gain = 1.0
1875  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1876  elif gain <= 0:
1877  patchedGain = 1.0
1878  self.log.warn("Gain for amp %s == %g <= 0; setting to %f." %
1879  (amp.getName(), gain, patchedGain))
1880  gain = patchedGain
1881 
1882  if self.config.doEmpiricalReadNoise and overscanImage is None:
1883  self.log.info("Overscan is none for EmpiricalReadNoise.")
1884 
1885  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1886  stats = afwMath.StatisticsControl()
1887  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1888  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1889  self.log.info("Calculated empirical read noise for amp %s: %f.", amp.getName(), readNoise)
1890  else:
1891  readNoise = amp.getReadNoise()
1892 
1893  isrFunctions.updateVariance(
1894  maskedImage=ampExposure.getMaskedImage(),
1895  gain=gain,
1896  readNoise=readNoise,
1897  )
1898 
1899  def darkCorrection(self, exposure, darkExposure, invert=False):
1900  """!Apply dark correction in place.
1901 
1902  Parameters
1903  ----------
1904  exposure : `lsst.afw.image.Exposure`
1905  Exposure to process.
1906  darkExposure : `lsst.afw.image.Exposure`
1907  Dark exposure of the same size as ``exposure``.
1908  invert : `Bool`, optional
1909  If True, re-add the dark to an already corrected image.
1910 
1911  Raises
1912  ------
1913  RuntimeError
1914  Raised if either ``exposure`` or ``darkExposure`` do not
1915  have their dark time defined.
1916 
1917  See Also
1918  --------
1919  lsst.ip.isr.isrFunctions.darkCorrection
1920  """
1921  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1922  if math.isnan(expScale):
1923  raise RuntimeError("Exposure darktime is NAN.")
1924  if darkExposure.getInfo().getVisitInfo() is not None:
1925  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1926  else:
1927  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1928  # so getDarkTime() does not exist.
1929  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1930  darkScale = 1.0
1931 
1932  if math.isnan(darkScale):
1933  raise RuntimeError("Dark calib darktime is NAN.")
1934  isrFunctions.darkCorrection(
1935  maskedImage=exposure.getMaskedImage(),
1936  darkMaskedImage=darkExposure.getMaskedImage(),
1937  expScale=expScale,
1938  darkScale=darkScale,
1939  invert=invert,
1940  trimToFit=self.config.doTrimToMatchCalib
1941  )
1942 
1943  def doLinearize(self, detector):
1944  """!Check if linearization is needed for the detector cameraGeom.
1945 
1946  Checks config.doLinearize and the linearity type of the first
1947  amplifier.
1948 
1949  Parameters
1950  ----------
1951  detector : `lsst.afw.cameraGeom.Detector`
1952  Detector to get linearity type from.
1953 
1954  Returns
1955  -------
1956  doLinearize : `Bool`
1957  If True, linearization should be performed.
1958  """
1959  return self.config.doLinearize and \
1960  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1961 
1962  def flatCorrection(self, exposure, flatExposure, invert=False):
1963  """!Apply flat correction in place.
1964 
1965  Parameters
1966  ----------
1967  exposure : `lsst.afw.image.Exposure`
1968  Exposure to process.
1969  flatExposure : `lsst.afw.image.Exposure`
1970  Flat exposure of the same size as ``exposure``.
1971  invert : `Bool`, optional
1972  If True, unflatten an already flattened image.
1973 
1974  See Also
1975  --------
1976  lsst.ip.isr.isrFunctions.flatCorrection
1977  """
1978  isrFunctions.flatCorrection(
1979  maskedImage=exposure.getMaskedImage(),
1980  flatMaskedImage=flatExposure.getMaskedImage(),
1981  scalingType=self.config.flatScalingType,
1982  userScale=self.config.flatUserScale,
1983  invert=invert,
1984  trimToFit=self.config.doTrimToMatchCalib
1985  )
1986 
1987  def saturationDetection(self, exposure, amp):
1988  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1989 
1990  Parameters
1991  ----------
1992  exposure : `lsst.afw.image.Exposure`
1993  Exposure to process. Only the amplifier DataSec is processed.
1994  amp : `lsst.afw.table.AmpInfoCatalog`
1995  Amplifier detector data.
1996 
1997  See Also
1998  --------
1999  lsst.ip.isr.isrFunctions.makeThresholdMask
2000  """
2001  if not math.isnan(amp.getSaturation()):
2002  maskedImage = exposure.getMaskedImage()
2003  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2004  isrFunctions.makeThresholdMask(
2005  maskedImage=dataView,
2006  threshold=amp.getSaturation(),
2007  growFootprints=0,
2008  maskName=self.config.saturatedMaskName,
2009  )
2010 
2011  def saturationInterpolation(self, exposure):
2012  """!Interpolate over saturated pixels, in place.
2013 
2014  This method should be called after `saturationDetection`, to
2015  ensure that the saturated pixels have been identified in the
2016  SAT mask. It should also be called after `assembleCcd`, since
2017  saturated regions may cross amplifier boundaries.
2018 
2019  Parameters
2020  ----------
2021  exposure : `lsst.afw.image.Exposure`
2022  Exposure to process.
2023 
2024  See Also
2025  --------
2026  lsst.ip.isr.isrTask.saturationDetection
2027  lsst.ip.isr.isrFunctions.interpolateFromMask
2028  """
2029  isrFunctions.interpolateFromMask(
2030  maskedImage=exposure.getMaskedImage(),
2031  fwhm=self.config.fwhm,
2032  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2033  maskNameList=list(self.config.saturatedMaskName),
2034  )
2035 
2036  def suspectDetection(self, exposure, amp):
2037  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2038 
2039  Parameters
2040  ----------
2041  exposure : `lsst.afw.image.Exposure`
2042  Exposure to process. Only the amplifier DataSec is processed.
2043  amp : `lsst.afw.table.AmpInfoCatalog`
2044  Amplifier detector data.
2045 
2046  See Also
2047  --------
2048  lsst.ip.isr.isrFunctions.makeThresholdMask
2049 
2050  Notes
2051  -----
2052  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2053  This is intended to indicate pixels that may be affected by unknown systematics;
2054  for example if non-linearity corrections above a certain level are unstable
2055  then that would be a useful value for suspectLevel. A value of `nan` indicates
2056  that no such level exists and no pixels are to be masked as suspicious.
2057  """
2058  suspectLevel = amp.getSuspectLevel()
2059  if math.isnan(suspectLevel):
2060  return
2061 
2062  maskedImage = exposure.getMaskedImage()
2063  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2064  isrFunctions.makeThresholdMask(
2065  maskedImage=dataView,
2066  threshold=suspectLevel,
2067  growFootprints=0,
2068  maskName=self.config.suspectMaskName,
2069  )
2070 
2071  def maskDefect(self, exposure, defectBaseList):
2072  """!Mask defects using mask plane "BAD", in place.
2073 
2074  Parameters
2075  ----------
2076  exposure : `lsst.afw.image.Exposure`
2077  Exposure to process.
2078  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2079  `lsst.afw.image.DefectBase`.
2080  List of defects to mask and interpolate.
2081 
2082  Notes
2083  -----
2084  Call this after CCD assembly, since defects may cross amplifier boundaries.
2085  """
2086  maskedImage = exposure.getMaskedImage()
2087  if not isinstance(defectBaseList, Defects):
2088  # Promotes DefectBase to Defect
2089  defectList = Defects(defectBaseList)
2090  else:
2091  defectList = defectBaseList
2092  defectList.maskPixels(maskedImage, maskName="BAD")
2093 
2094  if self.config.numEdgeSuspect > 0:
2095  goodBBox = maskedImage.getBBox()
2096  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2097  goodBBox.grow(-self.config.numEdgeSuspect)
2098  # Mask pixels outside goodBBox as SUSPECT
2099  SourceDetectionTask.setEdgeBits(
2100  maskedImage,
2101  goodBBox,
2102  maskedImage.getMask().getPlaneBitMask("SUSPECT")
2103  )
2104 
2105  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2106  """Mask and interpolate defects using mask plane "BAD", in place.
2107 
2108  Parameters
2109  ----------
2110  exposure : `lsst.afw.image.Exposure`
2111  Exposure to process.
2112  defectBaseList : `List` of `Defects`
2113 
2114  """
2115  self.maskDefects(exposure, defectBaseList)
2116  isrFunctions.interpolateFromMask(
2117  maskedImage=exposure.getMaskedImage(),
2118  fwhm=self.config.fwhm,
2119  growSaturatedFootprints=0,
2120  maskNameList=["BAD"],
2121  )
2122 
2123  def maskNan(self, exposure):
2124  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2125 
2126  Parameters
2127  ----------
2128  exposure : `lsst.afw.image.Exposure`
2129  Exposure to process.
2130 
2131  Notes
2132  -----
2133  We mask over all NaNs, including those that are masked with
2134  other bits (because those may or may not be interpolated over
2135  later, and we want to remove all NaNs). Despite this
2136  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2137  the historical name.
2138  """
2139  maskedImage = exposure.getMaskedImage()
2140 
2141  # Find and mask NaNs
2142  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2143  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2144  numNans = maskNans(maskedImage, maskVal)
2145  self.metadata.set("NUMNANS", numNans)
2146  if numNans > 0:
2147  self.log.warn(f"There were {numNans} unmasked NaNs.")
2148 
2149  def maskAndInterpolateNan(self, exposure):
2150  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2151 
2152  Parameters
2153  ----------
2154  exposure : `lsst.afw.image.Exposure`
2155  Exposure to process.
2156 
2157  See Also
2158  --------
2159  lsst.ip.isr.isrTask.maskNan()
2160  """
2161  self.maskNan(exposure)
2162  isrFunctions.interpolateFromMask(
2163  maskedImage=exposure.getMaskedImage(),
2164  fwhm=self.config.fwhm,
2165  growSaturatedFootprints=0,
2166  maskNameList=["UNMASKEDNAN"],
2167  )
2168 
2169  def measureBackground(self, exposure, IsrQaConfig=None):
2170  """Measure the image background in subgrids, for quality control purposes.
2171 
2172  Parameters
2173  ----------
2174  exposure : `lsst.afw.image.Exposure`
2175  Exposure to process.
2176  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2177  Configuration object containing parameters on which background
2178  statistics and subgrids to use.
2179  """
2180  if IsrQaConfig is not None:
2181  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2182  IsrQaConfig.flatness.nIter)
2183  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2184  statsControl.setAndMask(maskVal)
2185  maskedImage = exposure.getMaskedImage()
2186  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2187  skyLevel = stats.getValue(afwMath.MEDIAN)
2188  skySigma = stats.getValue(afwMath.STDEVCLIP)
2189  self.log.info("Flattened sky level: %f +/- %f." % (skyLevel, skySigma))
2190  metadata = exposure.getMetadata()
2191  metadata.set('SKYLEVEL', skyLevel)
2192  metadata.set('SKYSIGMA', skySigma)
2193 
2194  # calcluating flatlevel over the subgrids
2195  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2196  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2197  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2198  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2199  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2200  skyLevels = numpy.zeros((nX, nY))
2201 
2202  for j in range(nY):
2203  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2204  for i in range(nX):
2205  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2206 
2207  xLLC = xc - meshXHalf
2208  yLLC = yc - meshYHalf
2209  xURC = xc + meshXHalf - 1
2210  yURC = yc + meshYHalf - 1
2211 
2212  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2213  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2214 
2215  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2216 
2217  good = numpy.where(numpy.isfinite(skyLevels))
2218  skyMedian = numpy.median(skyLevels[good])
2219  flatness = (skyLevels[good] - skyMedian) / skyMedian
2220  flatness_rms = numpy.std(flatness)
2221  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2222 
2223  self.log.info("Measuring sky levels in %dx%d grids: %f." % (nX, nY, skyMedian))
2224  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f." %
2225  (nX, nY, flatness_pp, flatness_rms))
2226 
2227  metadata.set('FLATNESS_PP', float(flatness_pp))
2228  metadata.set('FLATNESS_RMS', float(flatness_rms))
2229  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2230  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2231  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2232 
2233  def roughZeroPoint(self, exposure):
2234  """Set an approximate magnitude zero point for the exposure.
2235 
2236  Parameters
2237  ----------
2238  exposure : `lsst.afw.image.Exposure`
2239  Exposure to process.
2240  """
2241  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2242  if filterName in self.config.fluxMag0T1:
2243  fluxMag0 = self.config.fluxMag0T1[filterName]
2244  else:
2245  self.log.warn("No rough magnitude zero point set for filter %s." % filterName)
2246  fluxMag0 = self.config.defaultFluxMag0T1
2247 
2248  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2249  if not expTime > 0: # handle NaN as well as <= 0
2250  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2251  return
2252 
2253  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2254  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2255 
2256  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2257  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2258 
2259  Parameters
2260  ----------
2261  ccdExposure : `lsst.afw.image.Exposure`
2262  Exposure to process.
2263  fpPolygon : `lsst.afw.geom.Polygon`
2264  Polygon in focal plane coordinates.
2265  """
2266  # Get ccd corners in focal plane coordinates
2267  ccd = ccdExposure.getDetector()
2268  fpCorners = ccd.getCorners(FOCAL_PLANE)
2269  ccdPolygon = Polygon(fpCorners)
2270 
2271  # Get intersection of ccd corners with fpPolygon
2272  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2273 
2274  # Transform back to pixel positions and build new polygon
2275  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2276  validPolygon = Polygon(ccdPoints)
2277  ccdExposure.getInfo().setValidPolygon(validPolygon)
2278 
2279  @contextmanager
2280  def flatContext(self, exp, flat, dark=None):
2281  """Context manager that applies and removes flats and darks,
2282  if the task is configured to apply them.
2283 
2284  Parameters
2285  ----------
2286  exp : `lsst.afw.image.Exposure`
2287  Exposure to process.
2288  flat : `lsst.afw.image.Exposure`
2289  Flat exposure the same size as ``exp``.
2290  dark : `lsst.afw.image.Exposure`, optional
2291  Dark exposure the same size as ``exp``.
2292 
2293  Yields
2294  ------
2295  exp : `lsst.afw.image.Exposure`
2296  The flat and dark corrected exposure.
2297  """
2298  if self.config.doDark and dark is not None:
2299  self.darkCorrection(exp, dark)
2300  if self.config.doFlat:
2301  self.flatCorrection(exp, flat)
2302  try:
2303  yield exp
2304  finally:
2305  if self.config.doFlat:
2306  self.flatCorrection(exp, flat, invert=True)
2307  if self.config.doDark and dark is not None:
2308  self.darkCorrection(exp, dark, invert=True)
2309 
2310  def debugView(self, exposure, stepname):
2311  """Utility function to examine ISR exposure at different stages.
2312 
2313  Parameters
2314  ----------
2315  exposure : `lsst.afw.image.Exposure`
2316  Exposure to view.
2317  stepname : `str`
2318  State of processing to view.
2319  """
2320  frame = getDebugFrame(self._display, stepname)
2321  if frame:
2322  display = getDisplay(frame)
2323  display.scale('asinh', 'zscale')
2324  display.mtv(exposure)
2325  prompt = "Press Enter to continue [c]... "
2326  while True:
2327  ans = input(prompt).lower()
2328  if ans in ("", "c",):
2329  break
2330 
2331 
2332 class FakeAmp(object):
2333  """A Detector-like object that supports returning gain and saturation level
2334 
2335  This is used when the input exposure does not have a detector.
2336 
2337  Parameters
2338  ----------
2339  exposure : `lsst.afw.image.Exposure`
2340  Exposure to generate a fake amplifier for.
2341  config : `lsst.ip.isr.isrTaskConfig`
2342  Configuration to apply to the fake amplifier.
2343  """
2344 
2345  def __init__(self, exposure, config):
2346  self._bbox = exposure.getBBox(afwImage.LOCAL)
2348  self._gain = config.gain
2349  self._readNoise = config.readNoise
2350  self._saturation = config.saturation
2351 
2352  def getBBox(self):
2353  return self._bbox
2354 
2355  def getRawBBox(self):
2356  return self._bbox
2357 
2358  def getHasRawInfo(self):
2359  return True # but see getRawHorizontalOverscanBBox()
2360 
2362  return self._RawHorizontalOverscanBBox
2363 
2364  def getGain(self):
2365  return self._gain
2366 
2367  def getReadNoise(self):
2368  return self._readNoise
2369 
2370  def getSaturation(self):
2371  return self._saturation
2372 
2373  def getSuspectLevel(self):
2374  return float("NaN")
2375 
2376 
2377 class RunIsrConfig(pexConfig.Config):
2378  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2379 
2380 
2381 class RunIsrTask(pipeBase.CmdLineTask):
2382  """Task to wrap the default IsrTask to allow it to be retargeted.
2383 
2384  The standard IsrTask can be called directly from a command line
2385  program, but doing so removes the ability of the task to be
2386  retargeted. As most cameras override some set of the IsrTask
2387  methods, this would remove those data-specific methods in the
2388  output post-ISR images. This wrapping class fixes the issue,
2389  allowing identical post-ISR images to be generated by both the
2390  processCcd and isrTask code.
2391  """
2392  ConfigClass = RunIsrConfig
2393  _DefaultName = "runIsr"
2394 
2395  def __init__(self, *args, **kwargs):
2396  super().__init__(*args, **kwargs)
2397  self.makeSubtask("isr")
2398 
2399  def runDataRef(self, dataRef):
2400  """
2401  Parameters
2402  ----------
2403  dataRef : `lsst.daf.persistence.ButlerDataRef`
2404  data reference of the detector data to be processed
2405 
2406  Returns
2407  -------
2408  result : `pipeBase.Struct`
2409  Result struct with component:
2410 
2411  - exposure : `lsst.afw.image.Exposure`
2412  Post-ISR processed exposure.
2413  """
2414  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:769
def runDataRef(self, sensorRef)
Definition: isrTask.py:1462
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2169
def debugView(self, exposure, stepname)
Definition: isrTask.py:2310
def __init__(self, kwargs)
Definition: isrTask.py:759
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1557
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:878
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:838
def runDataRef(self, dataRef)
Definition: isrTask.py:2399
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1026
def __init__(self, args, kwargs)
Definition: isrTask.py:2395
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:821
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2233
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2105
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2361
def maskNan(self, exposure)
Definition: isrTask.py:2123
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:808
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2071
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1712
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1604
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1962
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:875
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1516
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1899
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1943
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2256
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1641
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:831
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2280
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1850
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2149
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2036
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2011
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1987
def __init__(self, exposure, config)
Definition: isrTask.py:2345