lsst.ip.isr  18.1.0-13-gbfe7f7f
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 
56 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
57 
58 
59 class IsrTaskConfig(pexConfig.Config):
60  """Configuration parameters for IsrTask.
61 
62  Items are grouped in the order in which they are executed by the task.
63  """
64  # General ISR configuration
65 
66  # gen3 options
67  isrName = pexConfig.Field(
68  dtype=str,
69  doc="Name of ISR",
70  default="ISR",
71  )
72 
73  # input datasets
74  ccdExposure = pipeBase.InputDatasetField(
75  doc="Input exposure to process",
76  name="raw",
77  scalar=True,
78  storageClass="Exposure",
79  dimensions=["instrument", "exposure", "detector"],
80  )
81  camera = pipeBase.InputDatasetField(
82  doc="Input camera to construct complete exposures.",
83  name="camera",
84  scalar=True,
85  storageClass="Camera",
86  dimensions=["instrument", "calibration_label"],
87  )
88  bias = pipeBase.InputDatasetField(
89  doc="Input bias calibration.",
90  name="bias",
91  scalar=True,
92  storageClass="ImageF",
93  dimensions=["instrument", "calibration_label", "detector"],
94  )
95  dark = pipeBase.InputDatasetField(
96  doc="Input dark calibration.",
97  name="dark",
98  scalar=True,
99  storageClass="ImageF",
100  dimensions=["instrument", "calibration_label", "detector"],
101  )
102  flat = pipeBase.InputDatasetField(
103  doc="Input flat calibration.",
104  name="flat",
105  scalar=True,
106  storageClass="MaskedImageF",
107  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
108  )
109  bfKernel = pipeBase.InputDatasetField(
110  doc="Input brighter-fatter kernel.",
111  name="bfKernel",
112  scalar=True,
113  storageClass="NumpyArray",
114  dimensions=["instrument", "calibration_label"],
115  )
116  defects = pipeBase.InputDatasetField(
117  doc="Input defect tables.",
118  name="defects",
119  scalar=True,
120  storageClass="DefectsList",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  opticsTransmission = pipeBase.InputDatasetField(
124  doc="Transmission curve due to the optics.",
125  name="transmission_optics",
126  scalar=True,
127  storageClass="TransmissionCurve",
128  dimensions=["instrument", "calibration_label"],
129  )
130  filterTransmission = pipeBase.InputDatasetField(
131  doc="Transmission curve due to the filter.",
132  name="transmission_filter",
133  scalar=True,
134  storageClass="TransmissionCurve",
135  dimensions=["instrument", "physical_filter", "calibration_label"],
136  )
137  sensorTransmission = pipeBase.InputDatasetField(
138  doc="Transmission curve due to the sensor.",
139  name="transmission_sensor",
140  scalar=True,
141  storageClass="TransmissionCurve",
142  dimensions=["instrument", "calibration_label", "detector"],
143  )
144  atmosphereTransmission = pipeBase.InputDatasetField(
145  doc="Transmission curve due to the atmosphere.",
146  name="transmission_atmosphere",
147  scalar=True,
148  storageClass="TransmissionCurve",
149  dimensions=["instrument"],
150  )
151  illumMaskedImage = pipeBase.InputDatasetField(
152  doc="Input illumination correction.",
153  name="illum",
154  scalar=True,
155  storageClass="MaskedImageF",
156  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
157  )
158 
159  # output datasets
160  outputExposure = pipeBase.OutputDatasetField(
161  doc="Output ISR processed exposure.",
162  name="postISRCCD",
163  scalar=True,
164  storageClass="ExposureF",
165  dimensions=["instrument", "visit", "detector"],
166  )
167  outputOssThumbnail = pipeBase.OutputDatasetField(
168  doc="Output Overscan-subtracted thumbnail image.",
169  name="OssThumb",
170  scalar=True,
171  storageClass="Thumbnail",
172  dimensions=["instrument", "visit", "detector"],
173  )
174  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
175  doc="Output flat-corrected thumbnail image.",
176  name="FlattenedThumb",
177  scalar=True,
178  storageClass="TextStorage",
179  dimensions=["instrument", "visit", "detector"],
180  )
181 
182  quantum = pipeBase.QuantumConfig(
183  dimensions=["visit", "detector", "instrument"],
184  )
185 
186 
187  datasetType = pexConfig.Field(
188  dtype=str,
189  doc="Dataset type for input data; users will typically leave this alone, "
190  "but camera-specific ISR tasks will override it",
191  default="raw",
192  )
193 
194  fallbackFilterName = pexConfig.Field(
195  dtype=str,
196  doc="Fallback default filter name for calibrations.",
197  optional=True
198  )
199  expectWcs = pexConfig.Field(
200  dtype=bool,
201  default=True,
202  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
203  )
204  fwhm = pexConfig.Field(
205  dtype=float,
206  doc="FWHM of PSF in arcseconds.",
207  default=1.0,
208  )
209  qa = pexConfig.ConfigField(
210  dtype=isrQa.IsrQaConfig,
211  doc="QA related configuration options.",
212  )
213 
214  # Image conversion configuration
215  doConvertIntToFloat = pexConfig.Field(
216  dtype=bool,
217  doc="Convert integer raw images to floating point values?",
218  default=True,
219  )
220 
221  # Saturated pixel handling.
222  doSaturation = pexConfig.Field(
223  dtype=bool,
224  doc="Mask saturated pixels? NB: this is totally independent of the"
225  " interpolation option - this is ONLY setting the bits in the mask."
226  " To have them interpolated make sure doSaturationInterpolation=True",
227  default=True,
228  )
229  saturatedMaskName = pexConfig.Field(
230  dtype=str,
231  doc="Name of mask plane to use in saturation detection and interpolation",
232  default="SAT",
233  )
234  saturation = pexConfig.Field(
235  dtype=float,
236  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
237  default=float("NaN"),
238  )
239  growSaturationFootprintSize = pexConfig.Field(
240  dtype=int,
241  doc="Number of pixels by which to grow the saturation footprints",
242  default=1,
243  )
244 
245  # Suspect pixel handling.
246  doSuspect = pexConfig.Field(
247  dtype=bool,
248  doc="Mask suspect pixels?",
249  default=True,
250  )
251  suspectMaskName = pexConfig.Field(
252  dtype=str,
253  doc="Name of mask plane to use for suspect pixels",
254  default="SUSPECT",
255  )
256  numEdgeSuspect = pexConfig.Field(
257  dtype=int,
258  doc="Number of edge pixels to be flagged as untrustworthy.",
259  default=0,
260  )
261 
262  # Initial masking options.
263  doSetBadRegions = pexConfig.Field(
264  dtype=bool,
265  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
266  default=True,
267  )
268  badStatistic = pexConfig.ChoiceField(
269  dtype=str,
270  doc="How to estimate the average value for BAD regions.",
271  default='MEANCLIP',
272  allowed={
273  "MEANCLIP": "Correct using the (clipped) mean of good data",
274  "MEDIAN": "Correct using the median of the good data",
275  },
276  )
277 
278  # Overscan subtraction configuration.
279  doOverscan = pexConfig.Field(
280  dtype=bool,
281  doc="Do overscan subtraction?",
282  default=True,
283  )
284  overscanFitType = pexConfig.ChoiceField(
285  dtype=str,
286  doc="The method for fitting the overscan bias level.",
287  default='MEDIAN',
288  allowed={
289  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
290  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
291  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
292  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
293  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
294  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
295  "MEAN": "Correct using the mean of the overscan region",
296  "MEANCLIP": "Correct using a clipped mean of the overscan region",
297  "MEDIAN": "Correct using the median of the overscan region",
298  },
299  )
300  overscanOrder = pexConfig.Field(
301  dtype=int,
302  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
303  "or number of spline knots if overscan fit type is a spline."),
304  default=1,
305  )
306  overscanNumSigmaClip = pexConfig.Field(
307  dtype=float,
308  doc="Rejection threshold (sigma) for collapsing overscan before fit",
309  default=3.0,
310  )
311  overscanIsInt = pexConfig.Field(
312  dtype=bool,
313  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
314  default=True,
315  )
316  overscanNumLeadingColumnsToSkip = pexConfig.Field(
317  dtype=int,
318  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
319  default=0,
320  )
321  overscanNumTrailingColumnsToSkip = pexConfig.Field(
322  dtype=int,
323  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
324  default=0,
325  )
326  overscanMaxDev = pexConfig.Field(
327  dtype=float,
328  doc="Maximum deviation from the median for overscan",
329  default=1000.0, check=lambda x: x > 0
330  )
331  overscanBiasJump = pexConfig.Field(
332  dtype=bool,
333  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
334  default=False,
335  )
336  overscanBiasJumpKeyword = pexConfig.Field(
337  dtype=str,
338  doc="Header keyword containing information about devices.",
339  default="NO_SUCH_KEY",
340  )
341  overscanBiasJumpDevices = pexConfig.ListField(
342  dtype=str,
343  doc="List of devices that need piecewise overscan correction.",
344  default=(),
345  )
346  overscanBiasJumpLocation = pexConfig.Field(
347  dtype=int,
348  doc="Location of bias jump along y-axis.",
349  default=0,
350  )
351 
352  # Amplifier to CCD assembly configuration
353  doAssembleCcd = pexConfig.Field(
354  dtype=bool,
355  default=True,
356  doc="Assemble amp-level exposures into a ccd-level exposure?"
357  )
358  assembleCcd = pexConfig.ConfigurableField(
359  target=AssembleCcdTask,
360  doc="CCD assembly task",
361  )
362 
363  # General calibration configuration.
364  doAssembleIsrExposures = pexConfig.Field(
365  dtype=bool,
366  default=False,
367  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
368  )
369  doTrimToMatchCalib = pexConfig.Field(
370  dtype=bool,
371  default=False,
372  doc="Trim raw data to match calibration bounding boxes?"
373  )
374 
375  # Bias subtraction.
376  doBias = pexConfig.Field(
377  dtype=bool,
378  doc="Apply bias frame correction?",
379  default=True,
380  )
381  biasDataProductName = pexConfig.Field(
382  dtype=str,
383  doc="Name of the bias data product",
384  default="bias",
385  )
386 
387  # Variance construction
388  doVariance = pexConfig.Field(
389  dtype=bool,
390  doc="Calculate variance?",
391  default=True
392  )
393  gain = pexConfig.Field(
394  dtype=float,
395  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
396  default=float("NaN"),
397  )
398  readNoise = pexConfig.Field(
399  dtype=float,
400  doc="The read noise to use if no Detector is present in the Exposure",
401  default=0.0,
402  )
403  doEmpiricalReadNoise = pexConfig.Field(
404  dtype=bool,
405  default=False,
406  doc="Calculate empirical read noise instead of value from AmpInfo data?"
407  )
408 
409  # Linearization.
410  doLinearize = pexConfig.Field(
411  dtype=bool,
412  doc="Correct for nonlinearity of the detector's response?",
413  default=True,
414  )
415 
416  # Crosstalk.
417  doCrosstalk = pexConfig.Field(
418  dtype=bool,
419  doc="Apply intra-CCD crosstalk correction?",
420  default=False,
421  )
422  doCrosstalkBeforeAssemble = pexConfig.Field(
423  dtype=bool,
424  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
425  default=False,
426  )
427  crosstalk = pexConfig.ConfigurableField(
428  target=CrosstalkTask,
429  doc="Intra-CCD crosstalk correction",
430  )
431 
432  # Masking options.
433  doDefect = pexConfig.Field(
434  dtype=bool,
435  doc="Apply correction for CCD defects, e.g. hot pixels?",
436  default=True,
437  )
438  doNanMasking = pexConfig.Field(
439  dtype=bool,
440  doc="Mask NAN pixels?",
441  default=True,
442  )
443  doWidenSaturationTrails = pexConfig.Field(
444  dtype=bool,
445  doc="Widen bleed trails based on their width?",
446  default=True
447  )
448 
449  # Brighter-Fatter correction.
450  doBrighterFatter = pexConfig.Field(
451  dtype=bool,
452  default=False,
453  doc="Apply the brighter fatter correction"
454  )
455  brighterFatterLevel = pexConfig.ChoiceField(
456  dtype=str,
457  default="DETECTOR",
458  doc="The level at which to correct for brighter-fatter.",
459  allowed={
460  "AMP": "Every amplifier treated separately.",
461  "DETECTOR": "One kernel per detector",
462  }
463  )
464  brighterFatterKernelFile = pexConfig.Field(
465  dtype=str,
466  default='',
467  doc="Kernel file used for the brighter fatter correction"
468  )
469  brighterFatterMaxIter = pexConfig.Field(
470  dtype=int,
471  default=10,
472  doc="Maximum number of iterations for the brighter fatter correction"
473  )
474  brighterFatterThreshold = pexConfig.Field(
475  dtype=float,
476  default=1000,
477  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
478  " absolute value of the difference between the current corrected image and the one"
479  " from the previous iteration summed over all the pixels."
480  )
481  brighterFatterApplyGain = pexConfig.Field(
482  dtype=bool,
483  default=True,
484  doc="Should the gain be applied when applying the brighter fatter correction?"
485  )
486 
487  # Dark subtraction.
488  doDark = pexConfig.Field(
489  dtype=bool,
490  doc="Apply dark frame correction?",
491  default=True,
492  )
493  darkDataProductName = pexConfig.Field(
494  dtype=str,
495  doc="Name of the dark data product",
496  default="dark",
497  )
498 
499  # Camera-specific stray light removal.
500  doStrayLight = pexConfig.Field(
501  dtype=bool,
502  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
503  default=False,
504  )
505  strayLight = pexConfig.ConfigurableField(
506  target=StrayLightTask,
507  doc="y-band stray light correction"
508  )
509 
510  # Flat correction.
511  doFlat = pexConfig.Field(
512  dtype=bool,
513  doc="Apply flat field correction?",
514  default=True,
515  )
516  flatDataProductName = pexConfig.Field(
517  dtype=str,
518  doc="Name of the flat data product",
519  default="flat",
520  )
521  flatScalingType = pexConfig.ChoiceField(
522  dtype=str,
523  doc="The method for scaling the flat on the fly.",
524  default='USER',
525  allowed={
526  "USER": "Scale by flatUserScale",
527  "MEAN": "Scale by the inverse of the mean",
528  "MEDIAN": "Scale by the inverse of the median",
529  },
530  )
531  flatUserScale = pexConfig.Field(
532  dtype=float,
533  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
534  default=1.0,
535  )
536  doTweakFlat = pexConfig.Field(
537  dtype=bool,
538  doc="Tweak flats to match observed amplifier ratios?",
539  default=False
540  )
541 
542  # Amplifier normalization based on gains instead of using flats configuration.
543  doApplyGains = pexConfig.Field(
544  dtype=bool,
545  doc="Correct the amplifiers for their gains instead of applying flat correction",
546  default=False,
547  )
548  normalizeGains = pexConfig.Field(
549  dtype=bool,
550  doc="Normalize all the amplifiers in each CCD to have the same median value.",
551  default=False,
552  )
553 
554  # Fringe correction.
555  doFringe = pexConfig.Field(
556  dtype=bool,
557  doc="Apply fringe correction?",
558  default=True,
559  )
560  fringe = pexConfig.ConfigurableField(
561  target=FringeTask,
562  doc="Fringe subtraction task",
563  )
564  fringeAfterFlat = pexConfig.Field(
565  dtype=bool,
566  doc="Do fringe subtraction after flat-fielding?",
567  default=True,
568  )
569 
570  # Distortion model application.
571  doAddDistortionModel = pexConfig.Field(
572  dtype=bool,
573  doc="Apply a distortion model based on camera geometry to the WCS?",
574  default=True,
575  deprecated=("Camera geometry is incorporated when reading the raw files."
576  " This option no longer is used, and will be removed after v19.")
577  )
578 
579  # Initial CCD-level background statistics options.
580  doMeasureBackground = pexConfig.Field(
581  dtype=bool,
582  doc="Measure the background level on the reduced image?",
583  default=False,
584  )
585 
586  # Camera-specific masking configuration.
587  doCameraSpecificMasking = pexConfig.Field(
588  dtype=bool,
589  doc="Mask camera-specific bad regions?",
590  default=False,
591  )
592  masking = pexConfig.ConfigurableField(
593  target=MaskingTask,
594  doc="Masking task."
595  )
596 
597  # Interpolation options.
598 
599  doInterpolate = pexConfig.Field(
600  dtype=bool,
601  doc="Interpolate masked pixels?",
602  default=True,
603  )
604  doSaturationInterpolation = pexConfig.Field(
605  dtype=bool,
606  doc="Perform interpolation over pixels masked as saturated?"
607  " NB: This is independent of doSaturation; if that is False this plane"
608  " will likely be blank, resulting in a no-op here.",
609  default=True,
610  )
611  doNanInterpolation = pexConfig.Field(
612  dtype=bool,
613  doc="Perform interpolation over pixels masked as NaN?"
614  " NB: This is independent of doNanMasking; if that is False this plane"
615  " will likely be blank, resulting in a no-op here.",
616  default=True,
617  )
618  doNanInterpAfterFlat = pexConfig.Field(
619  dtype=bool,
620  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
621  "also have to interpolate them before flat-fielding."),
622  default=False,
623  )
624  maskListToInterpolate = pexConfig.ListField(
625  dtype=str,
626  doc="List of mask planes that should be interpolated.",
627  default=['SAT', 'BAD', 'UNMASKEDNAN'],
628  )
629  doSaveInterpPixels = pexConfig.Field(
630  dtype=bool,
631  doc="Save a copy of the pre-interpolated pixel values?",
632  default=False,
633  )
634 
635  # Default photometric calibration options.
636  fluxMag0T1 = pexConfig.DictField(
637  keytype=str,
638  itemtype=float,
639  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
640  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
641  ))
642  )
643  defaultFluxMag0T1 = pexConfig.Field(
644  dtype=float,
645  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
646  default=pow(10.0, 0.4*28.0)
647  )
648 
649  # Vignette correction configuration.
650  doVignette = pexConfig.Field(
651  dtype=bool,
652  doc="Apply vignetting parameters?",
653  default=False,
654  )
655  vignette = pexConfig.ConfigurableField(
656  target=VignetteTask,
657  doc="Vignetting task.",
658  )
659 
660  # Transmission curve configuration.
661  doAttachTransmissionCurve = pexConfig.Field(
662  dtype=bool,
663  default=False,
664  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
665  )
666  doUseOpticsTransmission = pexConfig.Field(
667  dtype=bool,
668  default=True,
669  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
670  )
671  doUseFilterTransmission = pexConfig.Field(
672  dtype=bool,
673  default=True,
674  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
675  )
676  doUseSensorTransmission = pexConfig.Field(
677  dtype=bool,
678  default=True,
679  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
680  )
681  doUseAtmosphereTransmission = pexConfig.Field(
682  dtype=bool,
683  default=True,
684  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
685  )
686 
687  # Illumination correction.
688  doIlluminationCorrection = pexConfig.Field(
689  dtype=bool,
690  default=False,
691  doc="Perform illumination correction?"
692  )
693  illuminationCorrectionDataProductName = pexConfig.Field(
694  dtype=str,
695  doc="Name of the illumination correction data product.",
696  default="illumcor",
697  )
698  illumScale = pexConfig.Field(
699  dtype=float,
700  doc="Scale factor for the illumination correction.",
701  default=1.0,
702  )
703  illumFilters = pexConfig.ListField(
704  dtype=str,
705  default=[],
706  doc="Only perform illumination correction for these filters."
707  )
708 
709  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
710  doWrite = pexConfig.Field(
711  dtype=bool,
712  doc="Persist postISRCCD?",
713  default=True,
714  )
715 
716  def validate(self):
717  super().validate()
718  if self.doFlat and self.doApplyGains:
719  raise ValueError("You may not specify both doFlat and doApplyGains")
720  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
721  self.config.maskListToInterpolate.append("SAT")
722  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
723  self.config.maskListToInterpolate.append("UNMASKEDNAN")
724 
725 
726 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
727  """Apply common instrument signature correction algorithms to a raw frame.
728 
729  The process for correcting imaging data is very similar from
730  camera to camera. This task provides a vanilla implementation of
731  doing these corrections, including the ability to turn certain
732  corrections off if they are not needed. The inputs to the primary
733  method, `run()`, are a raw exposure to be corrected and the
734  calibration data products. The raw input is a single chip sized
735  mosaic of all amps including overscans and other non-science
736  pixels. The method `runDataRef()` identifies and defines the
737  calibration data products, and is intended for use by a
738  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
739  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
740  subclassed for different camera, although the most camera specific
741  methods have been split into subtasks that can be redirected
742  appropriately.
743 
744  The __init__ method sets up the subtasks for ISR processing, using
745  the defaults from `lsst.ip.isr`.
746 
747  Parameters
748  ----------
749  args : `list`
750  Positional arguments passed to the Task constructor. None used at this time.
751  kwargs : `dict`, optional
752  Keyword arguments passed on to the Task constructor. None used at this time.
753  """
754  ConfigClass = IsrTaskConfig
755  _DefaultName = "isr"
756 
757  def __init__(self, **kwargs):
758  super().__init__(**kwargs)
759  self.makeSubtask("assembleCcd")
760  self.makeSubtask("crosstalk")
761  self.makeSubtask("strayLight")
762  self.makeSubtask("fringe")
763  self.makeSubtask("masking")
764  self.makeSubtask("vignette")
765 
766  @classmethod
767  def getInputDatasetTypes(cls, config):
768  inputTypeDict = super().getInputDatasetTypes(config)
769 
770  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
771  # need because the configuration tells us we will not be bothering with the
772  # correction that uses that IDT.
773  if config.doBias is not True:
774  inputTypeDict.pop("bias", None)
775  if config.doLinearize is not True:
776  inputTypeDict.pop("linearizer", None)
777  if config.doCrosstalk is not True:
778  inputTypeDict.pop("crosstalkSources", None)
779  if config.doBrighterFatter is not True:
780  inputTypeDict.pop("bfKernel", None)
781  if config.doDefect is not True:
782  inputTypeDict.pop("defects", None)
783  if config.doDark is not True:
784  inputTypeDict.pop("dark", None)
785  if config.doFlat is not True:
786  inputTypeDict.pop("flat", None)
787  if config.doAttachTransmissionCurve is not True:
788  inputTypeDict.pop("opticsTransmission", None)
789  inputTypeDict.pop("filterTransmission", None)
790  inputTypeDict.pop("sensorTransmission", None)
791  inputTypeDict.pop("atmosphereTransmission", None)
792  if config.doUseOpticsTransmission is not True:
793  inputTypeDict.pop("opticsTransmission", None)
794  if config.doUseFilterTransmission is not True:
795  inputTypeDict.pop("filterTransmission", None)
796  if config.doUseSensorTransmission is not True:
797  inputTypeDict.pop("sensorTransmission", None)
798  if config.doUseAtmosphereTransmission is not True:
799  inputTypeDict.pop("atmosphereTransmission", None)
800  if config.doIlluminationCorrection is not True:
801  inputTypeDict.pop("illumMaskedImage", None)
802 
803  return inputTypeDict
804 
805  @classmethod
806  def getOutputDatasetTypes(cls, config):
807  outputTypeDict = super().getOutputDatasetTypes(config)
808 
809  if config.qa.doThumbnailOss is not True:
810  outputTypeDict.pop("outputOssThumbnail", None)
811  if config.qa.doThumbnailFlattened is not True:
812  outputTypeDict.pop("outputFlattenedThumbnail", None)
813  if config.doWrite is not True:
814  outputTypeDict.pop("outputExposure", None)
815 
816  return outputTypeDict
817 
818  @classmethod
819  def getPrerequisiteDatasetTypes(cls, config):
820  # Input calibration datasets should not constrain the QuantumGraph
821  # (it'd be confusing if not having flats just silently resulted in no
822  # data being processed). Our nomenclature for that is that these are
823  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
824  names = set(cls.getInputDatasetTypes(config))
825  names.remove("ccdExposure")
826  return names
827 
828  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
829  try:
830  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
831  except Exception as e:
832  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
833  (inputDataIds, e))
834 
835  inputData['isGen3'] = True
836 
837  if self.config.doLinearize is True:
838  if 'linearizer' not in inputData.keys():
839  detector = inputData['camera'][inputData['detectorNum']]
840  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
841  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
842 
843  if inputData['defects'] is not None:
844  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
845  # masking expects a list of defects defined by their bounding box
846  if not isinstance(inputData["defects"], Defects):
847  inputData["defects"] = Defects.fromTable(inputData["defects"])
848 
849  # Broken: DM-17169
850  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
851  # fixed for non-HSC cameras in the future.
852  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
853  # if self.config.doCrosstalk else None)
854 
855  # Broken: DM-17152
856  # Fringes are not tested to be handled correctly by Gen3 butler.
857  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
858  # assembler=self.assembleCcd
859  # if self.config.doAssembleIsrExposures else None)
860  # if self.config.doFringe and
861  # self.fringe.checkFilter(inputData['ccdExposure'])
862  # else pipeBase.Struct(fringes=None))
863 
864  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
865 
866  def makeDatasetType(self, dsConfig):
867  return super().makeDatasetType(dsConfig)
868 
869  def readIsrData(self, dataRef, rawExposure):
870  """!Retrieve necessary frames for instrument signature removal.
871 
872  Pre-fetching all required ISR data products limits the IO
873  required by the ISR. Any conflict between the calibration data
874  available and that needed for ISR is also detected prior to
875  doing processing, allowing it to fail quickly.
876 
877  Parameters
878  ----------
879  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
880  Butler reference of the detector data to be processed
881  rawExposure : `afw.image.Exposure`
882  The raw exposure that will later be corrected with the
883  retrieved calibration data; should not be modified in this
884  method.
885 
886  Returns
887  -------
888  result : `lsst.pipe.base.Struct`
889  Result struct with components (which may be `None`):
890  - ``bias``: bias calibration frame (`afw.image.Exposure`)
891  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
892  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
893  - ``dark``: dark calibration frame (`afw.image.Exposure`)
894  - ``flat``: flat calibration frame (`afw.image.Exposure`)
895  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
896  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
897  - ``fringes``: `lsst.pipe.base.Struct` with components:
898  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
899  - ``seed``: random seed derived from the ccdExposureId for random
900  number generator (`uint32`).
901  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
902  A ``TransmissionCurve`` that represents the throughput of the optics,
903  to be evaluated in focal-plane coordinates.
904  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
905  A ``TransmissionCurve`` that represents the throughput of the filter
906  itself, to be evaluated in focal-plane coordinates.
907  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
908  A ``TransmissionCurve`` that represents the throughput of the sensor
909  itself, to be evaluated in post-assembly trimmed detector coordinates.
910  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
911  A ``TransmissionCurve`` that represents the throughput of the
912  atmosphere, assumed to be spatially constant.
913  - ``strayLightData`` : `object`
914  An opaque object containing calibration information for
915  stray-light correction. If `None`, no correction will be
916  performed.
917  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
918 
919  Raises
920  ------
921  NotImplementedError :
922  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
923  """
924  ccd = rawExposure.getDetector()
925  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
926  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
927  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
928  if self.config.doBias else None)
929  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
930  linearizer = (dataRef.get("linearizer", immediate=True)
931  if self.doLinearize(ccd) else None)
932  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
933  if self.config.doCrosstalk else None)
934  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
935  if self.config.doDark else None)
936  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
937  if self.config.doFlat else None)
938 
939  brighterFatterKernel = None
940  if self.config.doBrighterFatter is True:
941 
942  # Use the new-style cp_pipe version of the kernel is it exists.
943  try:
944  brighterFatterKernel = dataRef.get("brighterFatterKernel")
945  except NoResults:
946  # Fall back to the old-style numpy-ndarray style kernel if necessary.
947  try:
948  brighterFatterKernel = dataRef.get("bfKernel")
949  except NoResults:
950  brighterFatterKernel = None
951  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
952  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
953  # this detector, or raise an error.
954  if self.config.brighterFatterLevel == 'DETECTOR':
955  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
956  else:
957  # TODO DM-15631 for implementing this
958  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
959 
960  defectList = (dataRef.get("defects")
961  if self.config.doDefect else None)
962  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
963  if self.config.doAssembleIsrExposures else None)
964  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
965  else pipeBase.Struct(fringes=None))
966 
967  if self.config.doAttachTransmissionCurve:
968  opticsTransmission = (dataRef.get("transmission_optics")
969  if self.config.doUseOpticsTransmission else None)
970  filterTransmission = (dataRef.get("transmission_filter")
971  if self.config.doUseFilterTransmission else None)
972  sensorTransmission = (dataRef.get("transmission_sensor")
973  if self.config.doUseSensorTransmission else None)
974  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
975  if self.config.doUseAtmosphereTransmission else None)
976  else:
977  opticsTransmission = None
978  filterTransmission = None
979  sensorTransmission = None
980  atmosphereTransmission = None
981 
982  if self.config.doStrayLight:
983  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
984  else:
985  strayLightData = None
986 
987  illumMaskedImage = (self.getIsrExposure(dataRef,
988  self.config.illuminationCorrectionDataProductName).getMaskedImage()
989  if (self.config.doIlluminationCorrection and
990  filterName in self.config.illumFilters)
991  else None)
992 
993  # Struct should include only kwargs to run()
994  return pipeBase.Struct(bias=biasExposure,
995  linearizer=linearizer,
996  crosstalkSources=crosstalkSources,
997  dark=darkExposure,
998  flat=flatExposure,
999  bfKernel=brighterFatterKernel,
1000  defects=defectList,
1001  fringes=fringeStruct,
1002  opticsTransmission=opticsTransmission,
1003  filterTransmission=filterTransmission,
1004  sensorTransmission=sensorTransmission,
1005  atmosphereTransmission=atmosphereTransmission,
1006  strayLightData=strayLightData,
1007  illumMaskedImage=illumMaskedImage
1008  )
1009 
1010  @pipeBase.timeMethod
1011  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1012  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1013  opticsTransmission=None, filterTransmission=None,
1014  sensorTransmission=None, atmosphereTransmission=None,
1015  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1016  isGen3=False,
1017  ):
1018  """!Perform instrument signature removal on an exposure.
1019 
1020  Steps included in the ISR processing, in order performed, are:
1021  - saturation and suspect pixel masking
1022  - overscan subtraction
1023  - CCD assembly of individual amplifiers
1024  - bias subtraction
1025  - variance image construction
1026  - linearization of non-linear response
1027  - crosstalk masking
1028  - brighter-fatter correction
1029  - dark subtraction
1030  - fringe correction
1031  - stray light subtraction
1032  - flat correction
1033  - masking of known defects and camera specific features
1034  - vignette calculation
1035  - appending transmission curve and distortion model
1036 
1037  Parameters
1038  ----------
1039  ccdExposure : `lsst.afw.image.Exposure`
1040  The raw exposure that is to be run through ISR. The
1041  exposure is modified by this method.
1042  camera : `lsst.afw.cameraGeom.Camera`, optional
1043  The camera geometry for this exposure. Used to select the
1044  distortion model appropriate for this data.
1045  bias : `lsst.afw.image.Exposure`, optional
1046  Bias calibration frame.
1047  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1048  Functor for linearization.
1049  crosstalkSources : `list`, optional
1050  List of possible crosstalk sources.
1051  dark : `lsst.afw.image.Exposure`, optional
1052  Dark calibration frame.
1053  flat : `lsst.afw.image.Exposure`, optional
1054  Flat calibration frame.
1055  bfKernel : `numpy.ndarray`, optional
1056  Brighter-fatter kernel.
1057  defects : `lsst.meas.algorithms.Defects`, optional
1058  List of defects.
1059  fringes : `lsst.pipe.base.Struct`, optional
1060  Struct containing the fringe correction data, with
1061  elements:
1062  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1063  - ``seed``: random seed derived from the ccdExposureId for random
1064  number generator (`uint32`)
1065  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1066  A ``TransmissionCurve`` that represents the throughput of the optics,
1067  to be evaluated in focal-plane coordinates.
1068  filterTransmission : `lsst.afw.image.TransmissionCurve`
1069  A ``TransmissionCurve`` that represents the throughput of the filter
1070  itself, to be evaluated in focal-plane coordinates.
1071  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1072  A ``TransmissionCurve`` that represents the throughput of the sensor
1073  itself, to be evaluated in post-assembly trimmed detector coordinates.
1074  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1075  A ``TransmissionCurve`` that represents the throughput of the
1076  atmosphere, assumed to be spatially constant.
1077  detectorNum : `int`, optional
1078  The integer number for the detector to process.
1079  isGen3 : bool, optional
1080  Flag this call to run() as using the Gen3 butler environment.
1081  strayLightData : `object`, optional
1082  Opaque object containing calibration information for stray-light
1083  correction. If `None`, no correction will be performed.
1084  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1085  Illumination correction image.
1086 
1087  Returns
1088  -------
1089  result : `lsst.pipe.base.Struct`
1090  Result struct with component:
1091  - ``exposure`` : `afw.image.Exposure`
1092  The fully ISR corrected exposure.
1093  - ``outputExposure`` : `afw.image.Exposure`
1094  An alias for `exposure`
1095  - ``ossThumb`` : `numpy.ndarray`
1096  Thumbnail image of the exposure after overscan subtraction.
1097  - ``flattenedThumb`` : `numpy.ndarray`
1098  Thumbnail image of the exposure after flat-field correction.
1099 
1100  Raises
1101  ------
1102  RuntimeError
1103  Raised if a configuration option is set to True, but the
1104  required calibration data has not been specified.
1105 
1106  Notes
1107  -----
1108  The current processed exposure can be viewed by setting the
1109  appropriate lsstDebug entries in the `debug.display`
1110  dictionary. The names of these entries correspond to some of
1111  the IsrTaskConfig Boolean options, with the value denoting the
1112  frame to use. The exposure is shown inside the matching
1113  option check and after the processing of that step has
1114  finished. The steps with debug points are:
1115 
1116  doAssembleCcd
1117  doBias
1118  doCrosstalk
1119  doBrighterFatter
1120  doDark
1121  doFringe
1122  doStrayLight
1123  doFlat
1124 
1125  In addition, setting the "postISRCCD" entry displays the
1126  exposure after all ISR processing has finished.
1127 
1128  """
1129 
1130  if isGen3 is True:
1131  # Gen3 currently cannot automatically do configuration overrides.
1132  # DM-15257 looks to discuss this issue.
1133 
1134  self.config.doFringe = False
1135 
1136  # Configure input exposures;
1137  if detectorNum is None:
1138  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1139 
1140  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1141  bias = self.ensureExposure(bias, camera, detectorNum)
1142  dark = self.ensureExposure(dark, camera, detectorNum)
1143  flat = self.ensureExposure(flat, camera, detectorNum)
1144  else:
1145  if isinstance(ccdExposure, ButlerDataRef):
1146  return self.runDataRef(ccdExposure)
1147 
1148  ccd = ccdExposure.getDetector()
1149  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1150 
1151  if not ccd:
1152  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1153  ccd = [FakeAmp(ccdExposure, self.config)]
1154 
1155  # Validate Input
1156  if self.config.doBias and bias is None:
1157  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1158  if self.doLinearize(ccd) and linearizer is None:
1159  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1160  if self.config.doBrighterFatter and bfKernel is None:
1161  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1162  if self.config.doDark and dark is None:
1163  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1164  if self.config.doFlat and flat is None:
1165  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1166  if self.config.doDefect and defects is None:
1167  raise RuntimeError("Must supply defects if config.doDefect=True.")
1168  if (self.config.doFringe and filterName in self.fringe.config.filters and
1169  fringes.fringes is None):
1170  # The `fringes` object needs to be a pipeBase.Struct, as
1171  # we use it as a `dict` for the parameters of
1172  # `FringeTask.run()`. The `fringes.fringes` `list` may
1173  # not be `None` if `doFringe=True`. Otherwise, raise.
1174  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1175  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1176  illumMaskedImage is None):
1177  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1178 
1179  # Begin ISR processing.
1180  if self.config.doConvertIntToFloat:
1181  self.log.info("Converting exposure to floating point values.")
1182  ccdExposure = self.convertIntToFloat(ccdExposure)
1183 
1184  # Amplifier level processing.
1185  overscans = []
1186  for amp in ccd:
1187  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1188  if ccdExposure.getBBox().contains(amp.getBBox()):
1189  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1190  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1191 
1192  if self.config.doOverscan and not badAmp:
1193  # Overscan correction on amp-by-amp basis.
1194  overscanResults = self.overscanCorrection(ccdExposure, amp)
1195  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1196  if overscanResults is not None and \
1197  self.config.qa is not None and self.config.qa.saveStats is True:
1198  if isinstance(overscanResults.overscanFit, float):
1199  qaMedian = overscanResults.overscanFit
1200  qaStdev = float("NaN")
1201  else:
1202  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1203  afwMath.MEDIAN | afwMath.STDEVCLIP)
1204  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1205  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1206 
1207  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1208  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1209  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1210  amp.getName(), qaMedian, qaStdev)
1211  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1212  else:
1213  if badAmp:
1214  self.log.warn("Amplifier %s is bad.", amp.getName())
1215  overscanResults = None
1216 
1217  overscans.append(overscanResults if overscanResults is not None else None)
1218  else:
1219  self.log.info("Skipped OSCAN for %s.", amp.getName())
1220 
1221  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1222  self.log.info("Applying crosstalk correction.")
1223  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1224  self.debugView(ccdExposure, "doCrosstalk")
1225 
1226  if self.config.doAssembleCcd:
1227  self.log.info("Assembling CCD from amplifiers.")
1228  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1229 
1230  if self.config.expectWcs and not ccdExposure.getWcs():
1231  self.log.warn("No WCS found in input exposure.")
1232  self.debugView(ccdExposure, "doAssembleCcd")
1233 
1234  ossThumb = None
1235  if self.config.qa.doThumbnailOss:
1236  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1237 
1238  if self.config.doBias:
1239  self.log.info("Applying bias correction.")
1240  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1241  trimToFit=self.config.doTrimToMatchCalib)
1242  self.debugView(ccdExposure, "doBias")
1243 
1244  if self.config.doVariance:
1245  for amp, overscanResults in zip(ccd, overscans):
1246  if ccdExposure.getBBox().contains(amp.getBBox()):
1247  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1248  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1249  if overscanResults is not None:
1250  self.updateVariance(ampExposure, amp,
1251  overscanImage=overscanResults.overscanImage)
1252  else:
1253  self.updateVariance(ampExposure, amp,
1254  overscanImage=None)
1255  if self.config.qa is not None and self.config.qa.saveStats is True:
1256  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1257  afwMath.MEDIAN | afwMath.STDEVCLIP)
1258  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1259  qaStats.getValue(afwMath.MEDIAN))
1260  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1261  qaStats.getValue(afwMath.STDEVCLIP))
1262  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1263  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1264  qaStats.getValue(afwMath.STDEVCLIP))
1265 
1266  if self.doLinearize(ccd):
1267  self.log.info("Applying linearizer.")
1268  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1269 
1270  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1271  self.log.info("Applying crosstalk correction.")
1272  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1273  self.debugView(ccdExposure, "doCrosstalk")
1274 
1275  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1276  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1277  if self.config.doDefect:
1278  self.log.info("Masking defects.")
1279  self.maskDefect(ccdExposure, defects)
1280 
1281  if self.config.numEdgeSuspect > 0:
1282  self.log.info("Masking edges as SUSPECT.")
1283  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1284  maskPlane="SUSPECT")
1285 
1286  if self.config.doNanMasking:
1287  self.log.info("Masking NAN value pixels.")
1288  self.maskNan(ccdExposure)
1289 
1290  if self.config.doWidenSaturationTrails:
1291  self.log.info("Widening saturation trails.")
1292  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1293 
1294  if self.config.doCameraSpecificMasking:
1295  self.log.info("Masking regions for camera specific reasons.")
1296  self.masking.run(ccdExposure)
1297 
1298  if self.config.doBrighterFatter:
1299  # We need to apply flats and darks before we can interpolate, and we
1300  # need to interpolate before we do B-F, but we do B-F without the
1301  # flats and darks applied so we can work in units of electrons or holes.
1302  # This context manager applies and then removes the darks and flats.
1303  #
1304  # We also do not want to interpolate values here, so operate on temporary
1305  # images so we can apply only the BF-correction and roll back the
1306  # interpolation.
1307  interpExp = ccdExposure.clone()
1308  with self.flatContext(interpExp, flat, dark):
1309  isrFunctions.interpolateFromMask(
1310  maskedImage=interpExp.getMaskedImage(),
1311  fwhm=self.config.fwhm,
1312  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1313  maskNameList=self.config.maskListToInterpolate
1314  )
1315  bfExp = interpExp.clone()
1316 
1317  self.log.info("Applying brighter fatter correction.")
1318  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1319  self.config.brighterFatterMaxIter,
1320  self.config.brighterFatterThreshold,
1321  self.config.brighterFatterApplyGain
1322  )
1323  if bfResults[1] == self.config.brighterFatterMaxIter:
1324  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1325  bfResults[0])
1326  else:
1327  self.log.info("Finished brighter fatter correction in %d iterations.",
1328  bfResults[1])
1329  image = ccdExposure.getMaskedImage().getImage()
1330  bfCorr = bfExp.getMaskedImage().getImage()
1331  bfCorr -= interpExp.getMaskedImage().getImage()
1332  image += bfCorr
1333 
1334  # Applying the brighter-fatter correction applies a
1335  # convolution to the science image. At the edges this
1336  # convolution may not have sufficient valid pixels to
1337  # produce a valid correction. Mark pixels within the size
1338  # of the brighter-fatter kernel as EDGE to warn of this
1339  # fact.
1340  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1341  maskPlane="EDGE")
1342  self.log.warn("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1343 
1344  self.debugView(ccdExposure, "doBrighterFatter")
1345 
1346  if self.config.doDark:
1347  self.log.info("Applying dark correction.")
1348  self.darkCorrection(ccdExposure, dark)
1349  self.debugView(ccdExposure, "doDark")
1350 
1351  if self.config.doFringe and not self.config.fringeAfterFlat:
1352  self.log.info("Applying fringe correction before flat.")
1353  self.fringe.run(ccdExposure, **fringes.getDict())
1354  self.debugView(ccdExposure, "doFringe")
1355 
1356  if self.config.doStrayLight:
1357  if strayLightData is not None:
1358  self.log.info("Applying stray light correction.")
1359  self.strayLight.run(ccdExposure, strayLightData)
1360  self.debugView(ccdExposure, "doStrayLight")
1361  else:
1362  self.log.debug("Skipping stray light correction: no data found for this image.")
1363 
1364  if self.config.doFlat:
1365  self.log.info("Applying flat correction.")
1366  self.flatCorrection(ccdExposure, flat)
1367  self.debugView(ccdExposure, "doFlat")
1368 
1369  if self.config.doApplyGains:
1370  self.log.info("Applying gain correction instead of flat.")
1371  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1372 
1373  if self.config.doFringe and self.config.fringeAfterFlat:
1374  self.log.info("Applying fringe correction after flat.")
1375  self.fringe.run(ccdExposure, **fringes.getDict())
1376 
1377  if self.config.doVignette:
1378  self.log.info("Constructing Vignette polygon.")
1379  self.vignettePolygon = self.vignette.run(ccdExposure)
1380 
1381  if self.config.vignette.doWriteVignettePolygon:
1382  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1383 
1384  if self.config.doAttachTransmissionCurve:
1385  self.log.info("Adding transmission curves.")
1386  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1387  filterTransmission=filterTransmission,
1388  sensorTransmission=sensorTransmission,
1389  atmosphereTransmission=atmosphereTransmission)
1390 
1391  flattenedThumb = None
1392  if self.config.qa.doThumbnailFlattened:
1393  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1394 
1395  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1396  self.log.info("Performing illumination correction.")
1397  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1398  illumMaskedImage, illumScale=self.config.illumScale,
1399  trimToFit=self.config.doTrimToMatchCalib)
1400 
1401  preInterpExp = None
1402  if self.config.doSaveInterpPixels:
1403  preInterpExp = ccdExposure.clone()
1404 
1405  # Reset and interpolate bad pixels.
1406  #
1407  # Large contiguous bad regions (which should have the BAD mask
1408  # bit set) should have their values set to the image median.
1409  # This group should include defects and bad amplifiers. As the
1410  # area covered by these defects are large, there's little
1411  # reason to expect that interpolation would provide a more
1412  # useful value.
1413  #
1414  # Smaller defects can be safely interpolated after the larger
1415  # regions have had their pixel values reset. This ensures
1416  # that the remaining defects adjacent to bad amplifiers (as an
1417  # example) do not attempt to interpolate extreme values.
1418  if self.config.doSetBadRegions:
1419  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1420  if badPixelCount > 0:
1421  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1422 
1423  if self.config.doInterpolate:
1424  self.log.info("Interpolating masked pixels.")
1425  isrFunctions.interpolateFromMask(
1426  maskedImage=ccdExposure.getMaskedImage(),
1427  fwhm=self.config.fwhm,
1428  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1429  maskNameList=list(self.config.maskListToInterpolate)
1430  )
1431 
1432  self.roughZeroPoint(ccdExposure)
1433 
1434  if self.config.doMeasureBackground:
1435  self.log.info("Measuring background level.")
1436  self.measureBackground(ccdExposure, self.config.qa)
1437 
1438  if self.config.qa is not None and self.config.qa.saveStats is True:
1439  for amp in ccd:
1440  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1441  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1442  afwMath.MEDIAN | afwMath.STDEVCLIP)
1443  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1444  qaStats.getValue(afwMath.MEDIAN))
1445  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1446  qaStats.getValue(afwMath.STDEVCLIP))
1447  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1448  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1449  qaStats.getValue(afwMath.STDEVCLIP))
1450 
1451  self.debugView(ccdExposure, "postISRCCD")
1452 
1453  return pipeBase.Struct(
1454  exposure=ccdExposure,
1455  ossThumb=ossThumb,
1456  flattenedThumb=flattenedThumb,
1457 
1458  preInterpolatedExposure=preInterpExp,
1459  outputExposure=ccdExposure,
1460  outputOssThumbnail=ossThumb,
1461  outputFlattenedThumbnail=flattenedThumb,
1462  )
1463 
1464  @pipeBase.timeMethod
1465  def runDataRef(self, sensorRef):
1466  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1467 
1468  This method contains the `CmdLineTask` interface to the ISR
1469  processing. All IO is handled here, freeing the `run()` method
1470  to manage only pixel-level calculations. The steps performed
1471  are:
1472  - Read in necessary detrending/isr/calibration data.
1473  - Process raw exposure in `run()`.
1474  - Persist the ISR-corrected exposure as "postISRCCD" if
1475  config.doWrite=True.
1476 
1477  Parameters
1478  ----------
1479  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1480  DataRef of the detector data to be processed
1481 
1482  Returns
1483  -------
1484  result : `lsst.pipe.base.Struct`
1485  Result struct with component:
1486  - ``exposure`` : `afw.image.Exposure`
1487  The fully ISR corrected exposure.
1488 
1489  Raises
1490  ------
1491  RuntimeError
1492  Raised if a configuration option is set to True, but the
1493  required calibration data does not exist.
1494 
1495  """
1496  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1497 
1498  ccdExposure = sensorRef.get(self.config.datasetType)
1499 
1500  camera = sensorRef.get("camera")
1501  isrData = self.readIsrData(sensorRef, ccdExposure)
1502 
1503  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1504 
1505  if self.config.doWrite:
1506  sensorRef.put(result.exposure, "postISRCCD")
1507  if result.preInterpolatedExposure is not None:
1508  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1509  if result.ossThumb is not None:
1510  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1511  if result.flattenedThumb is not None:
1512  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1513 
1514  return result
1515 
1516  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1517  """!Retrieve a calibration dataset for removing instrument signature.
1518 
1519  Parameters
1520  ----------
1521 
1522  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1523  DataRef of the detector data to find calibration datasets
1524  for.
1525  datasetType : `str`
1526  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1527  immediate : `Bool`
1528  If True, disable butler proxies to enable error handling
1529  within this routine.
1530 
1531  Returns
1532  -------
1533  exposure : `lsst.afw.image.Exposure`
1534  Requested calibration frame.
1535 
1536  Raises
1537  ------
1538  RuntimeError
1539  Raised if no matching calibration frame can be found.
1540  """
1541  try:
1542  exp = dataRef.get(datasetType, immediate=immediate)
1543  except Exception as exc1:
1544  if not self.config.fallbackFilterName:
1545  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1546  try:
1547  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1548  except Exception as exc2:
1549  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1550  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1551  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1552 
1553  if self.config.doAssembleIsrExposures:
1554  exp = self.assembleCcd.assembleCcd(exp)
1555  return exp
1556 
1557  def ensureExposure(self, inputExp, camera, detectorNum):
1558  """Ensure that the data returned by Butler is a fully constructed exposure.
1559 
1560  ISR requires exposure-level image data for historical reasons, so if we did
1561  not recieve that from Butler, construct it from what we have, modifying the
1562  input in place.
1563 
1564  Parameters
1565  ----------
1566  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1567  `lsst.afw.image.ImageF`
1568  The input data structure obtained from Butler.
1569  camera : `lsst.afw.cameraGeom.camera`
1570  The camera associated with the image. Used to find the appropriate
1571  detector.
1572  detectorNum : `int`
1573  The detector this exposure should match.
1574 
1575  Returns
1576  -------
1577  inputExp : `lsst.afw.image.Exposure`
1578  The re-constructed exposure, with appropriate detector parameters.
1579 
1580  Raises
1581  ------
1582  TypeError
1583  Raised if the input data cannot be used to construct an exposure.
1584  """
1585  if isinstance(inputExp, afwImage.DecoratedImageU):
1586  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1587  elif isinstance(inputExp, afwImage.ImageF):
1588  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1589  elif isinstance(inputExp, afwImage.MaskedImageF):
1590  inputExp = afwImage.makeExposure(inputExp)
1591  elif isinstance(inputExp, afwImage.Exposure):
1592  pass
1593  elif inputExp is None:
1594  # Assume this will be caught by the setup if it is a problem.
1595  return inputExp
1596  else:
1597  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1598  (type(inputExp), ))
1599 
1600  if inputExp.getDetector() is None:
1601  inputExp.setDetector(camera[detectorNum])
1602 
1603  return inputExp
1604 
1605  def convertIntToFloat(self, exposure):
1606  """Convert exposure image from uint16 to float.
1607 
1608  If the exposure does not need to be converted, the input is
1609  immediately returned. For exposures that are converted to use
1610  floating point pixels, the variance is set to unity and the
1611  mask to zero.
1612 
1613  Parameters
1614  ----------
1615  exposure : `lsst.afw.image.Exposure`
1616  The raw exposure to be converted.
1617 
1618  Returns
1619  -------
1620  newexposure : `lsst.afw.image.Exposure`
1621  The input ``exposure``, converted to floating point pixels.
1622 
1623  Raises
1624  ------
1625  RuntimeError
1626  Raised if the exposure type cannot be converted to float.
1627 
1628  """
1629  if isinstance(exposure, afwImage.ExposureF):
1630  # Nothing to be done
1631  self.log.debug("Exposure already of type float.")
1632  return exposure
1633  if not hasattr(exposure, "convertF"):
1634  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1635 
1636  newexposure = exposure.convertF()
1637  newexposure.variance[:] = 1
1638  newexposure.mask[:] = 0x0
1639 
1640  return newexposure
1641 
1642  def maskAmplifier(self, ccdExposure, amp, defects):
1643  """Identify bad amplifiers, saturated and suspect pixels.
1644 
1645  Parameters
1646  ----------
1647  ccdExposure : `lsst.afw.image.Exposure`
1648  Input exposure to be masked.
1649  amp : `lsst.afw.table.AmpInfoCatalog`
1650  Catalog of parameters defining the amplifier on this
1651  exposure to mask.
1652  defects : `lsst.meas.algorithms.Defects`
1653  List of defects. Used to determine if the entire
1654  amplifier is bad.
1655 
1656  Returns
1657  -------
1658  badAmp : `Bool`
1659  If this is true, the entire amplifier area is covered by
1660  defects and unusable.
1661 
1662  """
1663  maskedImage = ccdExposure.getMaskedImage()
1664 
1665  badAmp = False
1666 
1667  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1668  # comparison with current defects definition.
1669  if defects is not None:
1670  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1671 
1672  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1673  # association with pixels in current ccdExposure).
1674  if badAmp:
1675  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1676  afwImage.PARENT)
1677  maskView = dataView.getMask()
1678  maskView |= maskView.getPlaneBitMask("BAD")
1679  del maskView
1680  return badAmp
1681 
1682  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1683  # Saturation and suspect pixels can be masked now, though.
1684  limits = dict()
1685  if self.config.doSaturation and not badAmp:
1686  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1687  if self.config.doSuspect and not badAmp:
1688  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1689  if math.isfinite(self.config.saturation):
1690  limits.update({self.config.saturatedMaskName: self.config.saturation})
1691 
1692  for maskName, maskThreshold in limits.items():
1693  if not math.isnan(maskThreshold):
1694  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1695  isrFunctions.makeThresholdMask(
1696  maskedImage=dataView,
1697  threshold=maskThreshold,
1698  growFootprints=0,
1699  maskName=maskName
1700  )
1701 
1702  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1703  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1704  afwImage.PARENT)
1705  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1706  self.config.suspectMaskName])
1707  if numpy.all(maskView.getArray() & maskVal > 0):
1708  badAmp = True
1709  maskView |= maskView.getPlaneBitMask("BAD")
1710 
1711  return badAmp
1712 
1713  def overscanCorrection(self, ccdExposure, amp):
1714  """Apply overscan correction in place.
1715 
1716  This method does initial pixel rejection of the overscan
1717  region. The overscan can also be optionally segmented to
1718  allow for discontinuous overscan responses to be fit
1719  separately. The actual overscan subtraction is performed by
1720  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1721  which is called here after the amplifier is preprocessed.
1722 
1723  Parameters
1724  ----------
1725  ccdExposure : `lsst.afw.image.Exposure`
1726  Exposure to have overscan correction performed.
1727  amp : `lsst.afw.table.AmpInfoCatalog`
1728  The amplifier to consider while correcting the overscan.
1729 
1730  Returns
1731  -------
1732  overscanResults : `lsst.pipe.base.Struct`
1733  Result struct with components:
1734  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1735  Value or fit subtracted from the amplifier image data.
1736  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1737  Value or fit subtracted from the overscan image data.
1738  - ``overscanImage`` : `lsst.afw.image.Image`
1739  Image of the overscan region with the overscan
1740  correction applied. This quantity is used to estimate
1741  the amplifier read noise empirically.
1742 
1743  Raises
1744  ------
1745  RuntimeError
1746  Raised if the ``amp`` does not contain raw pixel information.
1747 
1748  See Also
1749  --------
1750  lsst.ip.isr.isrFunctions.overscanCorrection
1751  """
1752  if not amp.getHasRawInfo():
1753  raise RuntimeError("This method must be executed on an amp with raw information.")
1754 
1755  if amp.getRawHorizontalOverscanBBox().isEmpty():
1756  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1757  return None
1758 
1759  statControl = afwMath.StatisticsControl()
1760  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1761 
1762  # Determine the bounding boxes
1763  dataBBox = amp.getRawDataBBox()
1764  oscanBBox = amp.getRawHorizontalOverscanBBox()
1765  dx0 = 0
1766  dx1 = 0
1767 
1768  prescanBBox = amp.getRawPrescanBBox()
1769  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1770  dx0 += self.config.overscanNumLeadingColumnsToSkip
1771  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1772  else:
1773  dx0 += self.config.overscanNumTrailingColumnsToSkip
1774  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1775 
1776  # Determine if we need to work on subregions of the amplifier and overscan.
1777  imageBBoxes = []
1778  overscanBBoxes = []
1779 
1780  if ((self.config.overscanBiasJump and
1781  self.config.overscanBiasJumpLocation) and
1782  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1783  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1784  self.config.overscanBiasJumpDevices)):
1785  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1786  yLower = self.config.overscanBiasJumpLocation
1787  yUpper = dataBBox.getHeight() - yLower
1788  else:
1789  yUpper = self.config.overscanBiasJumpLocation
1790  yLower = dataBBox.getHeight() - yUpper
1791 
1792  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1793  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1794  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1795  lsst.geom.Extent2I(dx0, 0),
1796  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1797  yLower)))
1798 
1799  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1800  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1801  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1802  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1803  yUpper)))
1804  else:
1805  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1806  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1807  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1808  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1809  oscanBBox.getHeight())))
1810 
1811  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1812  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1813  ampImage = ccdExposure.maskedImage[imageBBox]
1814  overscanImage = ccdExposure.maskedImage[overscanBBox]
1815 
1816  overscanArray = overscanImage.image.array
1817  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1818  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1819  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1820 
1821  statControl = afwMath.StatisticsControl()
1822  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1823 
1824  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1825  overscanImage=overscanImage,
1826  fitType=self.config.overscanFitType,
1827  order=self.config.overscanOrder,
1828  collapseRej=self.config.overscanNumSigmaClip,
1829  statControl=statControl,
1830  overscanIsInt=self.config.overscanIsInt
1831  )
1832 
1833  # Measure average overscan levels and record them in the metadata.
1834  levelStat = afwMath.MEDIAN
1835  sigmaStat = afwMath.STDEVCLIP
1836 
1837  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1838  self.config.qa.flatness.nIter)
1839  metadata = ccdExposure.getMetadata()
1840  ampNum = amp.getName()
1841  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1842  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1843  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1844  else:
1845  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1846  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1847  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1848 
1849  return overscanResults
1850 
1851  def updateVariance(self, ampExposure, amp, overscanImage=None):
1852  """Set the variance plane using the amplifier gain and read noise
1853 
1854  The read noise is calculated from the ``overscanImage`` if the
1855  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1856  the value from the amplifier data is used.
1857 
1858  Parameters
1859  ----------
1860  ampExposure : `lsst.afw.image.Exposure`
1861  Exposure to process.
1862  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1863  Amplifier detector data.
1864  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1865  Image of overscan, required only for empirical read noise.
1866 
1867  See also
1868  --------
1869  lsst.ip.isr.isrFunctions.updateVariance
1870  """
1871  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1872  gain = amp.getGain()
1873 
1874  if math.isnan(gain):
1875  gain = 1.0
1876  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1877  elif gain <= 0:
1878  patchedGain = 1.0
1879  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1880  amp.getName(), gain, patchedGain)
1881  gain = patchedGain
1882 
1883  if self.config.doEmpiricalReadNoise and overscanImage is None:
1884  self.log.info("Overscan is none for EmpiricalReadNoise.")
1885 
1886  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1887  stats = afwMath.StatisticsControl()
1888  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1889  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1890  self.log.info("Calculated empirical read noise for amp %s: %f.",
1891  amp.getName(), readNoise)
1892  else:
1893  readNoise = amp.getReadNoise()
1894 
1895  isrFunctions.updateVariance(
1896  maskedImage=ampExposure.getMaskedImage(),
1897  gain=gain,
1898  readNoise=readNoise,
1899  )
1900 
1901  def darkCorrection(self, exposure, darkExposure, invert=False):
1902  """!Apply dark correction in place.
1903 
1904  Parameters
1905  ----------
1906  exposure : `lsst.afw.image.Exposure`
1907  Exposure to process.
1908  darkExposure : `lsst.afw.image.Exposure`
1909  Dark exposure of the same size as ``exposure``.
1910  invert : `Bool`, optional
1911  If True, re-add the dark to an already corrected image.
1912 
1913  Raises
1914  ------
1915  RuntimeError
1916  Raised if either ``exposure`` or ``darkExposure`` do not
1917  have their dark time defined.
1918 
1919  See Also
1920  --------
1921  lsst.ip.isr.isrFunctions.darkCorrection
1922  """
1923  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1924  if math.isnan(expScale):
1925  raise RuntimeError("Exposure darktime is NAN.")
1926  if darkExposure.getInfo().getVisitInfo() is not None:
1927  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1928  else:
1929  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1930  # so getDarkTime() does not exist.
1931  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1932  darkScale = 1.0
1933 
1934  if math.isnan(darkScale):
1935  raise RuntimeError("Dark calib darktime is NAN.")
1936  isrFunctions.darkCorrection(
1937  maskedImage=exposure.getMaskedImage(),
1938  darkMaskedImage=darkExposure.getMaskedImage(),
1939  expScale=expScale,
1940  darkScale=darkScale,
1941  invert=invert,
1942  trimToFit=self.config.doTrimToMatchCalib
1943  )
1944 
1945  def doLinearize(self, detector):
1946  """!Check if linearization is needed for the detector cameraGeom.
1947 
1948  Checks config.doLinearize and the linearity type of the first
1949  amplifier.
1950 
1951  Parameters
1952  ----------
1953  detector : `lsst.afw.cameraGeom.Detector`
1954  Detector to get linearity type from.
1955 
1956  Returns
1957  -------
1958  doLinearize : `Bool`
1959  If True, linearization should be performed.
1960  """
1961  return self.config.doLinearize and \
1962  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1963 
1964  def flatCorrection(self, exposure, flatExposure, invert=False):
1965  """!Apply flat correction in place.
1966 
1967  Parameters
1968  ----------
1969  exposure : `lsst.afw.image.Exposure`
1970  Exposure to process.
1971  flatExposure : `lsst.afw.image.Exposure`
1972  Flat exposure of the same size as ``exposure``.
1973  invert : `Bool`, optional
1974  If True, unflatten an already flattened image.
1975 
1976  See Also
1977  --------
1978  lsst.ip.isr.isrFunctions.flatCorrection
1979  """
1980  isrFunctions.flatCorrection(
1981  maskedImage=exposure.getMaskedImage(),
1982  flatMaskedImage=flatExposure.getMaskedImage(),
1983  scalingType=self.config.flatScalingType,
1984  userScale=self.config.flatUserScale,
1985  invert=invert,
1986  trimToFit=self.config.doTrimToMatchCalib
1987  )
1988 
1989  def saturationDetection(self, exposure, amp):
1990  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1991 
1992  Parameters
1993  ----------
1994  exposure : `lsst.afw.image.Exposure`
1995  Exposure to process. Only the amplifier DataSec is processed.
1996  amp : `lsst.afw.table.AmpInfoCatalog`
1997  Amplifier detector data.
1998 
1999  See Also
2000  --------
2001  lsst.ip.isr.isrFunctions.makeThresholdMask
2002  """
2003  if not math.isnan(amp.getSaturation()):
2004  maskedImage = exposure.getMaskedImage()
2005  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2006  isrFunctions.makeThresholdMask(
2007  maskedImage=dataView,
2008  threshold=amp.getSaturation(),
2009  growFootprints=0,
2010  maskName=self.config.saturatedMaskName,
2011  )
2012 
2013  def saturationInterpolation(self, exposure):
2014  """!Interpolate over saturated pixels, in place.
2015 
2016  This method should be called after `saturationDetection`, to
2017  ensure that the saturated pixels have been identified in the
2018  SAT mask. It should also be called after `assembleCcd`, since
2019  saturated regions may cross amplifier boundaries.
2020 
2021  Parameters
2022  ----------
2023  exposure : `lsst.afw.image.Exposure`
2024  Exposure to process.
2025 
2026  See Also
2027  --------
2028  lsst.ip.isr.isrTask.saturationDetection
2029  lsst.ip.isr.isrFunctions.interpolateFromMask
2030  """
2031  isrFunctions.interpolateFromMask(
2032  maskedImage=exposure.getMaskedImage(),
2033  fwhm=self.config.fwhm,
2034  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2035  maskNameList=list(self.config.saturatedMaskName),
2036  )
2037 
2038  def suspectDetection(self, exposure, amp):
2039  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2040 
2041  Parameters
2042  ----------
2043  exposure : `lsst.afw.image.Exposure`
2044  Exposure to process. Only the amplifier DataSec is processed.
2045  amp : `lsst.afw.table.AmpInfoCatalog`
2046  Amplifier detector data.
2047 
2048  See Also
2049  --------
2050  lsst.ip.isr.isrFunctions.makeThresholdMask
2051 
2052  Notes
2053  -----
2054  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2055  This is intended to indicate pixels that may be affected by unknown systematics;
2056  for example if non-linearity corrections above a certain level are unstable
2057  then that would be a useful value for suspectLevel. A value of `nan` indicates
2058  that no such level exists and no pixels are to be masked as suspicious.
2059  """
2060  suspectLevel = amp.getSuspectLevel()
2061  if math.isnan(suspectLevel):
2062  return
2063 
2064  maskedImage = exposure.getMaskedImage()
2065  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2066  isrFunctions.makeThresholdMask(
2067  maskedImage=dataView,
2068  threshold=suspectLevel,
2069  growFootprints=0,
2070  maskName=self.config.suspectMaskName,
2071  )
2072 
2073  def maskDefect(self, exposure, defectBaseList):
2074  """!Mask defects using mask plane "BAD", in place.
2075 
2076  Parameters
2077  ----------
2078  exposure : `lsst.afw.image.Exposure`
2079  Exposure to process.
2080  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2081  `lsst.afw.image.DefectBase`.
2082  List of defects to mask.
2083 
2084  Notes
2085  -----
2086  Call this after CCD assembly, since defects may cross amplifier boundaries.
2087  """
2088  maskedImage = exposure.getMaskedImage()
2089  if not isinstance(defectBaseList, Defects):
2090  # Promotes DefectBase to Defect
2091  defectList = Defects(defectBaseList)
2092  else:
2093  defectList = defectBaseList
2094  defectList.maskPixels(maskedImage, maskName="BAD")
2095 
2096  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2097  """!Mask edge pixels with applicable mask plane.
2098 
2099  Parameters
2100  ----------
2101  exposure : `lsst.afw.image.Exposure`
2102  Exposure to process.
2103  numEdgePixels : `int`, optional
2104  Number of edge pixels to mask.
2105  maskPlane : `str`, optional
2106  Mask plane name to use.
2107  """
2108  maskedImage = exposure.getMaskedImage()
2109  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2110 
2111  if numEdgePixels > 0:
2112  goodBBox = maskedImage.getBBox()
2113  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2114  goodBBox.grow(-numEdgePixels)
2115  # Mask pixels outside goodBBox
2116  SourceDetectionTask.setEdgeBits(
2117  maskedImage,
2118  goodBBox,
2119  maskBitMask
2120  )
2121 
2122  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2123  """Mask and interpolate defects using mask plane "BAD", in place.
2124 
2125  Parameters
2126  ----------
2127  exposure : `lsst.afw.image.Exposure`
2128  Exposure to process.
2129  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2130  `lsst.afw.image.DefectBase`.
2131  List of defects to mask and interpolate.
2132 
2133  See Also
2134  --------
2135  lsst.ip.isr.isrTask.maskDefect()
2136  """
2137  self.maskDefect(exposure, defectBaseList)
2138  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2139  maskPlane="SUSPECT")
2140  isrFunctions.interpolateFromMask(
2141  maskedImage=exposure.getMaskedImage(),
2142  fwhm=self.config.fwhm,
2143  growSaturatedFootprints=0,
2144  maskNameList=["BAD"],
2145  )
2146 
2147  def maskNan(self, exposure):
2148  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2149 
2150  Parameters
2151  ----------
2152  exposure : `lsst.afw.image.Exposure`
2153  Exposure to process.
2154 
2155  Notes
2156  -----
2157  We mask over all NaNs, including those that are masked with
2158  other bits (because those may or may not be interpolated over
2159  later, and we want to remove all NaNs). Despite this
2160  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2161  the historical name.
2162  """
2163  maskedImage = exposure.getMaskedImage()
2164 
2165  # Find and mask NaNs
2166  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2167  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2168  numNans = maskNans(maskedImage, maskVal)
2169  self.metadata.set("NUMNANS", numNans)
2170  if numNans > 0:
2171  self.log.warn("There were %d unmasked NaNs.", numNans)
2172 
2173  def maskAndInterpolateNan(self, exposure):
2174  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2175 
2176  Parameters
2177  ----------
2178  exposure : `lsst.afw.image.Exposure`
2179  Exposure to process.
2180 
2181  See Also
2182  --------
2183  lsst.ip.isr.isrTask.maskNan()
2184  """
2185  self.maskNan(exposure)
2186  isrFunctions.interpolateFromMask(
2187  maskedImage=exposure.getMaskedImage(),
2188  fwhm=self.config.fwhm,
2189  growSaturatedFootprints=0,
2190  maskNameList=["UNMASKEDNAN"],
2191  )
2192 
2193  def measureBackground(self, exposure, IsrQaConfig=None):
2194  """Measure the image background in subgrids, for quality control purposes.
2195 
2196  Parameters
2197  ----------
2198  exposure : `lsst.afw.image.Exposure`
2199  Exposure to process.
2200  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2201  Configuration object containing parameters on which background
2202  statistics and subgrids to use.
2203  """
2204  if IsrQaConfig is not None:
2205  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2206  IsrQaConfig.flatness.nIter)
2207  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2208  statsControl.setAndMask(maskVal)
2209  maskedImage = exposure.getMaskedImage()
2210  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2211  skyLevel = stats.getValue(afwMath.MEDIAN)
2212  skySigma = stats.getValue(afwMath.STDEVCLIP)
2213  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2214  metadata = exposure.getMetadata()
2215  metadata.set('SKYLEVEL', skyLevel)
2216  metadata.set('SKYSIGMA', skySigma)
2217 
2218  # calcluating flatlevel over the subgrids
2219  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2220  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2221  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2222  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2223  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2224  skyLevels = numpy.zeros((nX, nY))
2225 
2226  for j in range(nY):
2227  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2228  for i in range(nX):
2229  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2230 
2231  xLLC = xc - meshXHalf
2232  yLLC = yc - meshYHalf
2233  xURC = xc + meshXHalf - 1
2234  yURC = yc + meshYHalf - 1
2235 
2236  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2237  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2238 
2239  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2240 
2241  good = numpy.where(numpy.isfinite(skyLevels))
2242  skyMedian = numpy.median(skyLevels[good])
2243  flatness = (skyLevels[good] - skyMedian) / skyMedian
2244  flatness_rms = numpy.std(flatness)
2245  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2246 
2247  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2248  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2249  nX, nY, flatness_pp, flatness_rms)
2250 
2251  metadata.set('FLATNESS_PP', float(flatness_pp))
2252  metadata.set('FLATNESS_RMS', float(flatness_rms))
2253  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2254  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2255  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2256 
2257  def roughZeroPoint(self, exposure):
2258  """Set an approximate magnitude zero point for the exposure.
2259 
2260  Parameters
2261  ----------
2262  exposure : `lsst.afw.image.Exposure`
2263  Exposure to process.
2264  """
2265  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2266  if filterName in self.config.fluxMag0T1:
2267  fluxMag0 = self.config.fluxMag0T1[filterName]
2268  else:
2269  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2270  fluxMag0 = self.config.defaultFluxMag0T1
2271 
2272  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2273  if not expTime > 0: # handle NaN as well as <= 0
2274  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2275  return
2276 
2277  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2278  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2279 
2280  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2281  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2282 
2283  Parameters
2284  ----------
2285  ccdExposure : `lsst.afw.image.Exposure`
2286  Exposure to process.
2287  fpPolygon : `lsst.afw.geom.Polygon`
2288  Polygon in focal plane coordinates.
2289  """
2290  # Get ccd corners in focal plane coordinates
2291  ccd = ccdExposure.getDetector()
2292  fpCorners = ccd.getCorners(FOCAL_PLANE)
2293  ccdPolygon = Polygon(fpCorners)
2294 
2295  # Get intersection of ccd corners with fpPolygon
2296  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2297 
2298  # Transform back to pixel positions and build new polygon
2299  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2300  validPolygon = Polygon(ccdPoints)
2301  ccdExposure.getInfo().setValidPolygon(validPolygon)
2302 
2303  @contextmanager
2304  def flatContext(self, exp, flat, dark=None):
2305  """Context manager that applies and removes flats and darks,
2306  if the task is configured to apply them.
2307 
2308  Parameters
2309  ----------
2310  exp : `lsst.afw.image.Exposure`
2311  Exposure to process.
2312  flat : `lsst.afw.image.Exposure`
2313  Flat exposure the same size as ``exp``.
2314  dark : `lsst.afw.image.Exposure`, optional
2315  Dark exposure the same size as ``exp``.
2316 
2317  Yields
2318  ------
2319  exp : `lsst.afw.image.Exposure`
2320  The flat and dark corrected exposure.
2321  """
2322  if self.config.doDark and dark is not None:
2323  self.darkCorrection(exp, dark)
2324  if self.config.doFlat:
2325  self.flatCorrection(exp, flat)
2326  try:
2327  yield exp
2328  finally:
2329  if self.config.doFlat:
2330  self.flatCorrection(exp, flat, invert=True)
2331  if self.config.doDark and dark is not None:
2332  self.darkCorrection(exp, dark, invert=True)
2333 
2334  def debugView(self, exposure, stepname):
2335  """Utility function to examine ISR exposure at different stages.
2336 
2337  Parameters
2338  ----------
2339  exposure : `lsst.afw.image.Exposure`
2340  Exposure to view.
2341  stepname : `str`
2342  State of processing to view.
2343  """
2344  frame = getDebugFrame(self._display, stepname)
2345  if frame:
2346  display = getDisplay(frame)
2347  display.scale('asinh', 'zscale')
2348  display.mtv(exposure)
2349  prompt = "Press Enter to continue [c]... "
2350  while True:
2351  ans = input(prompt).lower()
2352  if ans in ("", "c",):
2353  break
2354 
2355 
2356 class FakeAmp(object):
2357  """A Detector-like object that supports returning gain and saturation level
2358 
2359  This is used when the input exposure does not have a detector.
2360 
2361  Parameters
2362  ----------
2363  exposure : `lsst.afw.image.Exposure`
2364  Exposure to generate a fake amplifier for.
2365  config : `lsst.ip.isr.isrTaskConfig`
2366  Configuration to apply to the fake amplifier.
2367  """
2368 
2369  def __init__(self, exposure, config):
2370  self._bbox = exposure.getBBox(afwImage.LOCAL)
2372  self._gain = config.gain
2373  self._readNoise = config.readNoise
2374  self._saturation = config.saturation
2375 
2376  def getBBox(self):
2377  return self._bbox
2378 
2379  def getRawBBox(self):
2380  return self._bbox
2381 
2382  def getHasRawInfo(self):
2383  return True # but see getRawHorizontalOverscanBBox()
2384 
2386  return self._RawHorizontalOverscanBBox
2387 
2388  def getGain(self):
2389  return self._gain
2390 
2391  def getReadNoise(self):
2392  return self._readNoise
2393 
2394  def getSaturation(self):
2395  return self._saturation
2396 
2397  def getSuspectLevel(self):
2398  return float("NaN")
2399 
2400 
2401 class RunIsrConfig(pexConfig.Config):
2402  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2403 
2404 
2405 class RunIsrTask(pipeBase.CmdLineTask):
2406  """Task to wrap the default IsrTask to allow it to be retargeted.
2407 
2408  The standard IsrTask can be called directly from a command line
2409  program, but doing so removes the ability of the task to be
2410  retargeted. As most cameras override some set of the IsrTask
2411  methods, this would remove those data-specific methods in the
2412  output post-ISR images. This wrapping class fixes the issue,
2413  allowing identical post-ISR images to be generated by both the
2414  processCcd and isrTask code.
2415  """
2416  ConfigClass = RunIsrConfig
2417  _DefaultName = "runIsr"
2418 
2419  def __init__(self, *args, **kwargs):
2420  super().__init__(*args, **kwargs)
2421  self.makeSubtask("isr")
2422 
2423  def runDataRef(self, dataRef):
2424  """
2425  Parameters
2426  ----------
2427  dataRef : `lsst.daf.persistence.ButlerDataRef`
2428  data reference of the detector data to be processed
2429 
2430  Returns
2431  -------
2432  result : `pipeBase.Struct`
2433  Result struct with component:
2434 
2435  - exposure : `lsst.afw.image.Exposure`
2436  Post-ISR processed exposure.
2437  """
2438  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:767
def runDataRef(self, sensorRef)
Definition: isrTask.py:1465
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2193
def debugView(self, exposure, stepname)
Definition: isrTask.py:2334
def __init__(self, kwargs)
Definition: isrTask.py:757
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1557
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:869
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:828
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2096
def runDataRef(self, dataRef)
Definition: isrTask.py:2423
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1017
def __init__(self, args, kwargs)
Definition: isrTask.py:2419
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:819
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2257
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2122
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2385
def maskNan(self, exposure)
Definition: isrTask.py:2147
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:806
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2073
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1713
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1605
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1964
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:866
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1516
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1901
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1945
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2280
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1642
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2304
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1851
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2173
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2038
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2013
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1989
def __init__(self, exposure, config)
Definition: isrTask.py:2369