lsst.ip.isr  18.1.0-11-g28894db
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="Camera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TransmissionCurve",
148  dimensions=["instrument"],
149  )
150  illumMaskedImage = pipeBase.InputDatasetField(
151  doc="Input illumination correction.",
152  name="illum",
153  scalar=True,
154  storageClass="MaskedImageF",
155  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
156  )
157 
158  # output datasets
159  outputExposure = pipeBase.OutputDatasetField(
160  doc="Output ISR processed exposure.",
161  name="postISRCCD",
162  scalar=True,
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = pipeBase.OutputDatasetField(
167  doc="Output Overscan-subtracted thumbnail image.",
168  name="OssThumb",
169  scalar=True,
170  storageClass="Thumbnail",
171  dimensions=["instrument", "visit", "detector"],
172  )
173  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
174  doc="Output flat-corrected thumbnail image.",
175  name="FlattenedThumb",
176  scalar=True,
177  storageClass="TextStorage",
178  dimensions=["instrument", "visit", "detector"],
179  )
180 
181  quantum = pipeBase.QuantumConfig(
182  dimensions=["visit", "detector", "instrument"],
183  )
184 
185 
186  datasetType = pexConfig.Field(
187  dtype=str,
188  doc="Dataset type for input data; users will typically leave this alone, "
189  "but camera-specific ISR tasks will override it",
190  default="raw",
191  )
192 
193  fallbackFilterName = pexConfig.Field(
194  dtype=str,
195  doc="Fallback default filter name for calibrations.",
196  optional=True
197  )
198  expectWcs = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
202  )
203  fwhm = pexConfig.Field(
204  dtype=float,
205  doc="FWHM of PSF in arcseconds.",
206  default=1.0,
207  )
208  qa = pexConfig.ConfigField(
209  dtype=isrQa.IsrQaConfig,
210  doc="QA related configuration options.",
211  )
212 
213  # Image conversion configuration
214  doConvertIntToFloat = pexConfig.Field(
215  dtype=bool,
216  doc="Convert integer raw images to floating point values?",
217  default=True,
218  )
219 
220  # Saturated pixel handling.
221  doSaturation = pexConfig.Field(
222  dtype=bool,
223  doc="Mask saturated pixels? NB: this is totally independent of the"
224  " interpolation option - this is ONLY setting the bits in the mask."
225  " To have them interpolated make sure doSaturationInterpolation=True",
226  default=True,
227  )
228  saturatedMaskName = pexConfig.Field(
229  dtype=str,
230  doc="Name of mask plane to use in saturation detection and interpolation",
231  default="SAT",
232  )
233  saturation = pexConfig.Field(
234  dtype=float,
235  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
236  default=float("NaN"),
237  )
238  growSaturationFootprintSize = pexConfig.Field(
239  dtype=int,
240  doc="Number of pixels by which to grow the saturation footprints",
241  default=1,
242  )
243 
244  # Suspect pixel handling.
245  doSuspect = pexConfig.Field(
246  dtype=bool,
247  doc="Mask suspect pixels?",
248  default=True,
249  )
250  suspectMaskName = pexConfig.Field(
251  dtype=str,
252  doc="Name of mask plane to use for suspect pixels",
253  default="SUSPECT",
254  )
255  numEdgeSuspect = pexConfig.Field(
256  dtype=int,
257  doc="Number of edge pixels to be flagged as untrustworthy.",
258  default=0,
259  )
260 
261  # Initial masking options.
262  doSetBadRegions = pexConfig.Field(
263  dtype=bool,
264  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
265  default=True,
266  )
267  badStatistic = pexConfig.ChoiceField(
268  dtype=str,
269  doc="How to estimate the average value for BAD regions.",
270  default='MEANCLIP',
271  allowed={
272  "MEANCLIP": "Correct using the (clipped) mean of good data",
273  "MEDIAN": "Correct using the median of the good data",
274  },
275  )
276 
277  # Overscan subtraction configuration.
278  doOverscan = pexConfig.Field(
279  dtype=bool,
280  doc="Do overscan subtraction?",
281  default=True,
282  )
283  overscanFitType = pexConfig.ChoiceField(
284  dtype=str,
285  doc="The method for fitting the overscan bias level.",
286  default='MEDIAN',
287  allowed={
288  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
289  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
290  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
291  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
292  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
293  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
294  "MEAN": "Correct using the mean of the overscan region",
295  "MEANCLIP": "Correct using a clipped mean of the overscan region",
296  "MEDIAN": "Correct using the median of the overscan region",
297  },
298  )
299  overscanOrder = pexConfig.Field(
300  dtype=int,
301  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
302  "or number of spline knots if overscan fit type is a spline."),
303  default=1,
304  )
305  overscanNumSigmaClip = pexConfig.Field(
306  dtype=float,
307  doc="Rejection threshold (sigma) for collapsing overscan before fit",
308  default=3.0,
309  )
310  overscanIsInt = pexConfig.Field(
311  dtype=bool,
312  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
313  default=True,
314  )
315  overscanNumLeadingColumnsToSkip = pexConfig.Field(
316  dtype=int,
317  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
318  default=0,
319  )
320  overscanNumTrailingColumnsToSkip = pexConfig.Field(
321  dtype=int,
322  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
323  default=0,
324  )
325  overscanMaxDev = pexConfig.Field(
326  dtype=float,
327  doc="Maximum deviation from the median for overscan",
328  default=1000.0, check=lambda x: x > 0
329  )
330  overscanBiasJump = pexConfig.Field(
331  dtype=bool,
332  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
333  default=False,
334  )
335  overscanBiasJumpKeyword = pexConfig.Field(
336  dtype=str,
337  doc="Header keyword containing information about devices.",
338  default="NO_SUCH_KEY",
339  )
340  overscanBiasJumpDevices = pexConfig.ListField(
341  dtype=str,
342  doc="List of devices that need piecewise overscan correction.",
343  default=(),
344  )
345  overscanBiasJumpLocation = pexConfig.Field(
346  dtype=int,
347  doc="Location of bias jump along y-axis.",
348  default=0,
349  )
350 
351  # Amplifier to CCD assembly configuration
352  doAssembleCcd = pexConfig.Field(
353  dtype=bool,
354  default=True,
355  doc="Assemble amp-level exposures into a ccd-level exposure?"
356  )
357  assembleCcd = pexConfig.ConfigurableField(
358  target=AssembleCcdTask,
359  doc="CCD assembly task",
360  )
361 
362  # General calibration configuration.
363  doAssembleIsrExposures = pexConfig.Field(
364  dtype=bool,
365  default=False,
366  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
367  )
368  doTrimToMatchCalib = pexConfig.Field(
369  dtype=bool,
370  default=False,
371  doc="Trim raw data to match calibration bounding boxes?"
372  )
373 
374  # Bias subtraction.
375  doBias = pexConfig.Field(
376  dtype=bool,
377  doc="Apply bias frame correction?",
378  default=True,
379  )
380  biasDataProductName = pexConfig.Field(
381  dtype=str,
382  doc="Name of the bias data product",
383  default="bias",
384  )
385 
386  # Variance construction
387  doVariance = pexConfig.Field(
388  dtype=bool,
389  doc="Calculate variance?",
390  default=True
391  )
392  gain = pexConfig.Field(
393  dtype=float,
394  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
395  default=float("NaN"),
396  )
397  readNoise = pexConfig.Field(
398  dtype=float,
399  doc="The read noise to use if no Detector is present in the Exposure",
400  default=0.0,
401  )
402  doEmpiricalReadNoise = pexConfig.Field(
403  dtype=bool,
404  default=False,
405  doc="Calculate empirical read noise instead of value from AmpInfo data?"
406  )
407 
408  # Linearization.
409  doLinearize = pexConfig.Field(
410  dtype=bool,
411  doc="Correct for nonlinearity of the detector's response?",
412  default=True,
413  )
414 
415  # Crosstalk.
416  doCrosstalk = pexConfig.Field(
417  dtype=bool,
418  doc="Apply intra-CCD crosstalk correction?",
419  default=False,
420  )
421  doCrosstalkBeforeAssemble = pexConfig.Field(
422  dtype=bool,
423  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
424  default=True,
425  )
426  crosstalk = pexConfig.ConfigurableField(
427  target=CrosstalkTask,
428  doc="Intra-CCD crosstalk correction",
429  )
430 
431  # Masking options.
432  doDefect = pexConfig.Field(
433  dtype=bool,
434  doc="Apply correction for CCD defects, e.g. hot pixels?",
435  default=True,
436  )
437  doNanMasking = pexConfig.Field(
438  dtype=bool,
439  doc="Mask NAN pixels?",
440  default=True,
441  )
442  doWidenSaturationTrails = pexConfig.Field(
443  dtype=bool,
444  doc="Widen bleed trails based on their width?",
445  default=True
446  )
447 
448  # Brighter-Fatter correction.
449  doBrighterFatter = pexConfig.Field(
450  dtype=bool,
451  default=False,
452  doc="Apply the brighter fatter correction"
453  )
454  brighterFatterLevel = pexConfig.ChoiceField(
455  dtype=str,
456  default="DETECTOR",
457  doc="The level at which to correct for brighter-fatter.",
458  allowed={
459  "AMP": "Every amplifier treated separately.",
460  "DETECTOR": "One kernel per detector",
461  }
462  )
463  brighterFatterKernelFile = pexConfig.Field(
464  dtype=str,
465  default='',
466  doc="Kernel file used for the brighter fatter correction"
467  )
468  brighterFatterMaxIter = pexConfig.Field(
469  dtype=int,
470  default=10,
471  doc="Maximum number of iterations for the brighter fatter correction"
472  )
473  brighterFatterThreshold = pexConfig.Field(
474  dtype=float,
475  default=1000,
476  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
477  " absolute value of the difference between the current corrected image and the one"
478  " from the previous iteration summed over all the pixels."
479  )
480  brighterFatterApplyGain = pexConfig.Field(
481  dtype=bool,
482  default=True,
483  doc="Should the gain be applied when applying the brighter fatter correction?"
484  )
485 
486  # Dark subtraction.
487  doDark = pexConfig.Field(
488  dtype=bool,
489  doc="Apply dark frame correction?",
490  default=True,
491  )
492  darkDataProductName = pexConfig.Field(
493  dtype=str,
494  doc="Name of the dark data product",
495  default="dark",
496  )
497 
498  # Camera-specific stray light removal.
499  doStrayLight = pexConfig.Field(
500  dtype=bool,
501  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
502  default=False,
503  )
504  strayLight = pexConfig.ConfigurableField(
505  target=StrayLightTask,
506  doc="y-band stray light correction"
507  )
508 
509  # Flat correction.
510  doFlat = pexConfig.Field(
511  dtype=bool,
512  doc="Apply flat field correction?",
513  default=True,
514  )
515  flatDataProductName = pexConfig.Field(
516  dtype=str,
517  doc="Name of the flat data product",
518  default="flat",
519  )
520  flatScalingType = pexConfig.ChoiceField(
521  dtype=str,
522  doc="The method for scaling the flat on the fly.",
523  default='USER',
524  allowed={
525  "USER": "Scale by flatUserScale",
526  "MEAN": "Scale by the inverse of the mean",
527  "MEDIAN": "Scale by the inverse of the median",
528  },
529  )
530  flatUserScale = pexConfig.Field(
531  dtype=float,
532  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
533  default=1.0,
534  )
535  doTweakFlat = pexConfig.Field(
536  dtype=bool,
537  doc="Tweak flats to match observed amplifier ratios?",
538  default=False
539  )
540 
541  # Amplifier normalization based on gains instead of using flats configuration.
542  doApplyGains = pexConfig.Field(
543  dtype=bool,
544  doc="Correct the amplifiers for their gains instead of applying flat correction",
545  default=False,
546  )
547  normalizeGains = pexConfig.Field(
548  dtype=bool,
549  doc="Normalize all the amplifiers in each CCD to have the same median value.",
550  default=False,
551  )
552 
553  # Fringe correction.
554  doFringe = pexConfig.Field(
555  dtype=bool,
556  doc="Apply fringe correction?",
557  default=True,
558  )
559  fringe = pexConfig.ConfigurableField(
560  target=FringeTask,
561  doc="Fringe subtraction task",
562  )
563  fringeAfterFlat = pexConfig.Field(
564  dtype=bool,
565  doc="Do fringe subtraction after flat-fielding?",
566  default=True,
567  )
568 
569  # Distortion model application.
570  doAddDistortionModel = pexConfig.Field(
571  dtype=bool,
572  doc="Apply a distortion model based on camera geometry to the WCS?",
573  default=True,
574  )
575 
576  # Initial CCD-level background statistics options.
577  doMeasureBackground = pexConfig.Field(
578  dtype=bool,
579  doc="Measure the background level on the reduced image?",
580  default=False,
581  )
582 
583  # Camera-specific masking configuration.
584  doCameraSpecificMasking = pexConfig.Field(
585  dtype=bool,
586  doc="Mask camera-specific bad regions?",
587  default=False,
588  )
589  masking = pexConfig.ConfigurableField(
590  target=MaskingTask,
591  doc="Masking task."
592  )
593 
594  # Interpolation options.
595 
596  doInterpolate = pexConfig.Field(
597  dtype=bool,
598  doc="Interpolate masked pixels?",
599  default=True,
600  )
601  doSaturationInterpolation = pexConfig.Field(
602  dtype=bool,
603  doc="Perform interpolation over pixels masked as saturated?"
604  " NB: This is independent of doSaturation; if that is False this plane"
605  " will likely be blank, resulting in a no-op here.",
606  default=True,
607  )
608  doNanInterpolation = pexConfig.Field(
609  dtype=bool,
610  doc="Perform interpolation over pixels masked as NaN?"
611  " NB: This is independent of doNanMasking; if that is False this plane"
612  " will likely be blank, resulting in a no-op here.",
613  default=True,
614  )
615  doNanInterpAfterFlat = pexConfig.Field(
616  dtype=bool,
617  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
618  "also have to interpolate them before flat-fielding."),
619  default=False,
620  )
621  maskListToInterpolate = pexConfig.ListField(
622  dtype=str,
623  doc="List of mask planes that should be interpolated.",
624  default=['SAT', 'BAD', 'UNMASKEDNAN'],
625  )
626  doSaveInterpPixels = pexConfig.Field(
627  dtype=bool,
628  doc="Save a copy of the pre-interpolated pixel values?",
629  default=False,
630  )
631 
632  # Default photometric calibration options.
633  fluxMag0T1 = pexConfig.DictField(
634  keytype=str,
635  itemtype=float,
636  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
637  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
638  ))
639  )
640  defaultFluxMag0T1 = pexConfig.Field(
641  dtype=float,
642  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
643  default=pow(10.0, 0.4*28.0)
644  )
645 
646  # Vignette correction configuration.
647  doVignette = pexConfig.Field(
648  dtype=bool,
649  doc="Apply vignetting parameters?",
650  default=False,
651  )
652  vignette = pexConfig.ConfigurableField(
653  target=VignetteTask,
654  doc="Vignetting task.",
655  )
656 
657  # Transmission curve configuration.
658  doAttachTransmissionCurve = pexConfig.Field(
659  dtype=bool,
660  default=False,
661  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
662  )
663  doUseOpticsTransmission = pexConfig.Field(
664  dtype=bool,
665  default=True,
666  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
667  )
668  doUseFilterTransmission = pexConfig.Field(
669  dtype=bool,
670  default=True,
671  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
672  )
673  doUseSensorTransmission = pexConfig.Field(
674  dtype=bool,
675  default=True,
676  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
677  )
678  doUseAtmosphereTransmission = pexConfig.Field(
679  dtype=bool,
680  default=True,
681  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
682  )
683 
684  # Illumination correction.
685  doIlluminationCorrection = pexConfig.Field(
686  dtype=bool,
687  default=False,
688  doc="Perform illumination correction?"
689  )
690  illuminationCorrectionDataProductName = pexConfig.Field(
691  dtype=str,
692  doc="Name of the illumination correction data product.",
693  default="illumcor",
694  )
695  illumScale = pexConfig.Field(
696  dtype=float,
697  doc="Scale factor for the illumination correction.",
698  default=1.0,
699  )
700  illumFilters = pexConfig.ListField(
701  dtype=str,
702  default=[],
703  doc="Only perform illumination correction for these filters."
704  )
705 
706  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
707  doWrite = pexConfig.Field(
708  dtype=bool,
709  doc="Persist postISRCCD?",
710  default=True,
711  )
712 
713  def validate(self):
714  super().validate()
715  if self.doFlat and self.doApplyGains:
716  raise ValueError("You may not specify both doFlat and doApplyGains")
717  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
718  self.config.maskListToInterpolate.append("SAT")
719  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
720  self.config.maskListToInterpolate.append("UNMASKEDNAN")
721 
722 
723 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
724  """Apply common instrument signature correction algorithms to a raw frame.
725 
726  The process for correcting imaging data is very similar from
727  camera to camera. This task provides a vanilla implementation of
728  doing these corrections, including the ability to turn certain
729  corrections off if they are not needed. The inputs to the primary
730  method, `run()`, are a raw exposure to be corrected and the
731  calibration data products. The raw input is a single chip sized
732  mosaic of all amps including overscans and other non-science
733  pixels. The method `runDataRef()` identifies and defines the
734  calibration data products, and is intended for use by a
735  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
736  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
737  subclassed for different camera, although the most camera specific
738  methods have been split into subtasks that can be redirected
739  appropriately.
740 
741  The __init__ method sets up the subtasks for ISR processing, using
742  the defaults from `lsst.ip.isr`.
743 
744  Parameters
745  ----------
746  args : `list`
747  Positional arguments passed to the Task constructor. None used at this time.
748  kwargs : `dict`, optional
749  Keyword arguments passed on to the Task constructor. None used at this time.
750  """
751  ConfigClass = IsrTaskConfig
752  _DefaultName = "isr"
753 
754  def __init__(self, **kwargs):
755  super().__init__(**kwargs)
756  self.makeSubtask("assembleCcd")
757  self.makeSubtask("crosstalk")
758  self.makeSubtask("strayLight")
759  self.makeSubtask("fringe")
760  self.makeSubtask("masking")
761  self.makeSubtask("vignette")
762 
763  @classmethod
764  def getInputDatasetTypes(cls, config):
765  inputTypeDict = super().getInputDatasetTypes(config)
766 
767  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
768  # need because the configuration tells us we will not be bothering with the
769  # correction that uses that IDT.
770  if config.doBias is not True:
771  inputTypeDict.pop("bias", None)
772  if config.doLinearize is not True:
773  inputTypeDict.pop("linearizer", None)
774  if config.doCrosstalk is not True:
775  inputTypeDict.pop("crosstalkSources", None)
776  if config.doBrighterFatter is not True:
777  inputTypeDict.pop("bfKernel", None)
778  if config.doDefect is not True:
779  inputTypeDict.pop("defects", None)
780  if config.doDark is not True:
781  inputTypeDict.pop("dark", None)
782  if config.doFlat is not True:
783  inputTypeDict.pop("flat", None)
784  if config.doAttachTransmissionCurve is not True:
785  inputTypeDict.pop("opticsTransmission", None)
786  inputTypeDict.pop("filterTransmission", None)
787  inputTypeDict.pop("sensorTransmission", None)
788  inputTypeDict.pop("atmosphereTransmission", None)
789  if config.doUseOpticsTransmission is not True:
790  inputTypeDict.pop("opticsTransmission", None)
791  if config.doUseFilterTransmission is not True:
792  inputTypeDict.pop("filterTransmission", None)
793  if config.doUseSensorTransmission is not True:
794  inputTypeDict.pop("sensorTransmission", None)
795  if config.doUseAtmosphereTransmission is not True:
796  inputTypeDict.pop("atmosphereTransmission", None)
797  if config.doIlluminationCorrection is not True:
798  inputTypeDict.pop("illumMaskedImage", None)
799 
800  return inputTypeDict
801 
802  @classmethod
803  def getOutputDatasetTypes(cls, config):
804  outputTypeDict = super().getOutputDatasetTypes(config)
805 
806  if config.qa.doThumbnailOss is not True:
807  outputTypeDict.pop("outputOssThumbnail", None)
808  if config.qa.doThumbnailFlattened is not True:
809  outputTypeDict.pop("outputFlattenedThumbnail", None)
810  if config.doWrite is not True:
811  outputTypeDict.pop("outputExposure", None)
812 
813  return outputTypeDict
814 
815  @classmethod
816  def getPrerequisiteDatasetTypes(cls, config):
817  # Input calibration datasets should not constrain the QuantumGraph
818  # (it'd be confusing if not having flats just silently resulted in no
819  # data being processed). Our nomenclature for that is that these are
820  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
821  names = set(cls.getInputDatasetTypes(config))
822  names.remove("ccdExposure")
823  return names
824 
825  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
826  try:
827  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
828  except Exception as e:
829  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
830  (inputDataIds, e))
831 
832  inputData['isGen3'] = True
833 
834  if self.config.doLinearize is True:
835  if 'linearizer' not in inputData.keys():
836  detector = inputData['camera'][inputData['detectorNum']]
837  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
838  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
839 
840  if inputData['defects'] is not None:
841  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
842  # masking expects a list of defects defined by their bounding box
843  if not isinstance(inputData["defects"], Defects):
844  inputData["defects"] = Defects.fromTable(inputData["defects"])
845 
846  # Broken: DM-17169
847  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
848  # fixed for non-HSC cameras in the future.
849  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
850  # if self.config.doCrosstalk else None)
851 
852  # Broken: DM-17152
853  # Fringes are not tested to be handled correctly by Gen3 butler.
854  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
855  # assembler=self.assembleCcd
856  # if self.config.doAssembleIsrExposures else None)
857  # if self.config.doFringe and
858  # self.fringe.checkFilter(inputData['ccdExposure'])
859  # else pipeBase.Struct(fringes=None))
860 
861  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
862 
863  def makeDatasetType(self, dsConfig):
864  return super().makeDatasetType(dsConfig)
865 
866  def readIsrData(self, dataRef, rawExposure):
867  """!Retrieve necessary frames for instrument signature removal.
868 
869  Pre-fetching all required ISR data products limits the IO
870  required by the ISR. Any conflict between the calibration data
871  available and that needed for ISR is also detected prior to
872  doing processing, allowing it to fail quickly.
873 
874  Parameters
875  ----------
876  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
877  Butler reference of the detector data to be processed
878  rawExposure : `afw.image.Exposure`
879  The raw exposure that will later be corrected with the
880  retrieved calibration data; should not be modified in this
881  method.
882 
883  Returns
884  -------
885  result : `lsst.pipe.base.Struct`
886  Result struct with components (which may be `None`):
887  - ``bias``: bias calibration frame (`afw.image.Exposure`)
888  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
889  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
890  - ``dark``: dark calibration frame (`afw.image.Exposure`)
891  - ``flat``: flat calibration frame (`afw.image.Exposure`)
892  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
893  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
894  - ``fringes``: `lsst.pipe.base.Struct` with components:
895  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
896  - ``seed``: random seed derived from the ccdExposureId for random
897  number generator (`uint32`).
898  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
899  A ``TransmissionCurve`` that represents the throughput of the optics,
900  to be evaluated in focal-plane coordinates.
901  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
902  A ``TransmissionCurve`` that represents the throughput of the filter
903  itself, to be evaluated in focal-plane coordinates.
904  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
905  A ``TransmissionCurve`` that represents the throughput of the sensor
906  itself, to be evaluated in post-assembly trimmed detector coordinates.
907  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
908  A ``TransmissionCurve`` that represents the throughput of the
909  atmosphere, assumed to be spatially constant.
910  - ``strayLightData`` : `object`
911  An opaque object containing calibration information for
912  stray-light correction. If `None`, no correction will be
913  performed.
914  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
915 
916  Raises
917  ------
918  NotImplementedError :
919  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
920  """
921  ccd = rawExposure.getDetector()
922  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
923  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
924  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
925  if self.config.doBias else None)
926  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
927  linearizer = (dataRef.get("linearizer", immediate=True)
928  if self.doLinearize(ccd) else None)
929  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
930  if self.config.doCrosstalk else None)
931  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
932  if self.config.doDark else None)
933  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
934  if self.config.doFlat else None)
935 
936  brighterFatterKernel = None
937  if self.config.doBrighterFatter is True:
938 
939  # Use the new-style cp_pipe version of the kernel is it exists.
940  try:
941  brighterFatterKernel = dataRef.get("brighterFatterKernel")
942  except NoResults:
943  # Fall back to the old-style numpy-ndarray style kernel if necessary.
944  try:
945  brighterFatterKernel = dataRef.get("bfKernel")
946  except NoResults:
947  brighterFatterKernel = None
948  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
949  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
950  # this detector, or raise an error.
951  if self.config.brighterFatterLevel == 'DETECTOR':
952  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
953  else:
954  # TODO DM-15631 for implementing this
955  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
956 
957  defectList = (dataRef.get("defects")
958  if self.config.doDefect else None)
959  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
960  if self.config.doAssembleIsrExposures else None)
961  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
962  else pipeBase.Struct(fringes=None))
963 
964  if self.config.doAttachTransmissionCurve:
965  opticsTransmission = (dataRef.get("transmission_optics")
966  if self.config.doUseOpticsTransmission else None)
967  filterTransmission = (dataRef.get("transmission_filter")
968  if self.config.doUseFilterTransmission else None)
969  sensorTransmission = (dataRef.get("transmission_sensor")
970  if self.config.doUseSensorTransmission else None)
971  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
972  if self.config.doUseAtmosphereTransmission else None)
973  else:
974  opticsTransmission = None
975  filterTransmission = None
976  sensorTransmission = None
977  atmosphereTransmission = None
978 
979  if self.config.doStrayLight:
980  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
981  else:
982  strayLightData = None
983 
984  illumMaskedImage = (self.getIsrExposure(dataRef,
985  self.config.illuminationCorrectionDataProductName).getMaskedImage()
986  if (self.config.doIlluminationCorrection and
987  filterName in self.config.illumFilters)
988  else None)
989 
990  # Struct should include only kwargs to run()
991  return pipeBase.Struct(bias=biasExposure,
992  linearizer=linearizer,
993  crosstalkSources=crosstalkSources,
994  dark=darkExposure,
995  flat=flatExposure,
996  bfKernel=brighterFatterKernel,
997  defects=defectList,
998  fringes=fringeStruct,
999  opticsTransmission=opticsTransmission,
1000  filterTransmission=filterTransmission,
1001  sensorTransmission=sensorTransmission,
1002  atmosphereTransmission=atmosphereTransmission,
1003  strayLightData=strayLightData,
1004  illumMaskedImage=illumMaskedImage
1005  )
1006 
1007  @pipeBase.timeMethod
1008  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1009  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1010  opticsTransmission=None, filterTransmission=None,
1011  sensorTransmission=None, atmosphereTransmission=None,
1012  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1013  isGen3=False,
1014  ):
1015  """!Perform instrument signature removal on an exposure.
1016 
1017  Steps included in the ISR processing, in order performed, are:
1018  - saturation and suspect pixel masking
1019  - overscan subtraction
1020  - CCD assembly of individual amplifiers
1021  - bias subtraction
1022  - variance image construction
1023  - linearization of non-linear response
1024  - crosstalk masking
1025  - brighter-fatter correction
1026  - dark subtraction
1027  - fringe correction
1028  - stray light subtraction
1029  - flat correction
1030  - masking of known defects and camera specific features
1031  - vignette calculation
1032  - appending transmission curve and distortion model
1033 
1034  Parameters
1035  ----------
1036  ccdExposure : `lsst.afw.image.Exposure`
1037  The raw exposure that is to be run through ISR. The
1038  exposure is modified by this method.
1039  camera : `lsst.afw.cameraGeom.Camera`, optional
1040  The camera geometry for this exposure. Used to select the
1041  distortion model appropriate for this data.
1042  bias : `lsst.afw.image.Exposure`, optional
1043  Bias calibration frame.
1044  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1045  Functor for linearization.
1046  crosstalkSources : `list`, optional
1047  List of possible crosstalk sources.
1048  dark : `lsst.afw.image.Exposure`, optional
1049  Dark calibration frame.
1050  flat : `lsst.afw.image.Exposure`, optional
1051  Flat calibration frame.
1052  bfKernel : `numpy.ndarray`, optional
1053  Brighter-fatter kernel.
1054  defects : `lsst.meas.algorithms.Defects`, optional
1055  List of defects.
1056  fringes : `lsst.pipe.base.Struct`, optional
1057  Struct containing the fringe correction data, with
1058  elements:
1059  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1060  - ``seed``: random seed derived from the ccdExposureId for random
1061  number generator (`uint32`)
1062  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1063  A ``TransmissionCurve`` that represents the throughput of the optics,
1064  to be evaluated in focal-plane coordinates.
1065  filterTransmission : `lsst.afw.image.TransmissionCurve`
1066  A ``TransmissionCurve`` that represents the throughput of the filter
1067  itself, to be evaluated in focal-plane coordinates.
1068  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1069  A ``TransmissionCurve`` that represents the throughput of the sensor
1070  itself, to be evaluated in post-assembly trimmed detector coordinates.
1071  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1072  A ``TransmissionCurve`` that represents the throughput of the
1073  atmosphere, assumed to be spatially constant.
1074  detectorNum : `int`, optional
1075  The integer number for the detector to process.
1076  isGen3 : bool, optional
1077  Flag this call to run() as using the Gen3 butler environment.
1078  strayLightData : `object`, optional
1079  Opaque object containing calibration information for stray-light
1080  correction. If `None`, no correction will be performed.
1081  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1082  Illumination correction image.
1083 
1084  Returns
1085  -------
1086  result : `lsst.pipe.base.Struct`
1087  Result struct with component:
1088  - ``exposure`` : `afw.image.Exposure`
1089  The fully ISR corrected exposure.
1090  - ``outputExposure`` : `afw.image.Exposure`
1091  An alias for `exposure`
1092  - ``ossThumb`` : `numpy.ndarray`
1093  Thumbnail image of the exposure after overscan subtraction.
1094  - ``flattenedThumb`` : `numpy.ndarray`
1095  Thumbnail image of the exposure after flat-field correction.
1096 
1097  Raises
1098  ------
1099  RuntimeError
1100  Raised if a configuration option is set to True, but the
1101  required calibration data has not been specified.
1102 
1103  Notes
1104  -----
1105  The current processed exposure can be viewed by setting the
1106  appropriate lsstDebug entries in the `debug.display`
1107  dictionary. The names of these entries correspond to some of
1108  the IsrTaskConfig Boolean options, with the value denoting the
1109  frame to use. The exposure is shown inside the matching
1110  option check and after the processing of that step has
1111  finished. The steps with debug points are:
1112 
1113  doAssembleCcd
1114  doBias
1115  doCrosstalk
1116  doBrighterFatter
1117  doDark
1118  doFringe
1119  doStrayLight
1120  doFlat
1121 
1122  In addition, setting the "postISRCCD" entry displays the
1123  exposure after all ISR processing has finished.
1124 
1125  """
1126 
1127  if isGen3 is True:
1128  # Gen3 currently cannot automatically do configuration overrides.
1129  # DM-15257 looks to discuss this issue.
1130 
1131  self.config.doFringe = False
1132 
1133  # Configure input exposures;
1134  if detectorNum is None:
1135  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1136 
1137  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1138  bias = self.ensureExposure(bias, camera, detectorNum)
1139  dark = self.ensureExposure(dark, camera, detectorNum)
1140  flat = self.ensureExposure(flat, camera, detectorNum)
1141  else:
1142  if isinstance(ccdExposure, ButlerDataRef):
1143  return self.runDataRef(ccdExposure)
1144 
1145  ccd = ccdExposure.getDetector()
1146  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1147 
1148  if not ccd:
1149  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1150  ccd = [FakeAmp(ccdExposure, self.config)]
1151 
1152  # Validate Input
1153  if self.config.doBias and bias is None:
1154  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1155  if self.doLinearize(ccd) and linearizer is None:
1156  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1157  if self.config.doBrighterFatter and bfKernel is None:
1158  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1159  if self.config.doDark and dark is None:
1160  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1161  if self.config.doFlat and flat is None:
1162  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1163  if self.config.doDefect and defects is None:
1164  raise RuntimeError("Must supply defects if config.doDefect=True.")
1165  if self.config.doAddDistortionModel and camera is None:
1166  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1167  if (self.config.doFringe and filterName in self.fringe.config.filters and
1168  fringes.fringes is None):
1169  # The `fringes` object needs to be a pipeBase.Struct, as
1170  # we use it as a `dict` for the parameters of
1171  # `FringeTask.run()`. The `fringes.fringes` `list` may
1172  # not be `None` if `doFringe=True`. Otherwise, raise.
1173  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1174  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1175  illumMaskedImage is None):
1176  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1177 
1178  # Begin ISR processing.
1179  if self.config.doConvertIntToFloat:
1180  self.log.info("Converting exposure to floating point values.")
1181  ccdExposure = self.convertIntToFloat(ccdExposure)
1182 
1183  # Amplifier level processing.
1184  overscans = []
1185  for amp in ccd:
1186  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1187  if ccdExposure.getBBox().contains(amp.getBBox()):
1188  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1189  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1190 
1191  if self.config.doOverscan and not badAmp:
1192  # Overscan correction on amp-by-amp basis.
1193  overscanResults = self.overscanCorrection(ccdExposure, amp)
1194  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1195  if overscanResults is not None and \
1196  self.config.qa is not None and self.config.qa.saveStats is True:
1197  if isinstance(overscanResults.overscanFit, float):
1198  qaMedian = overscanResults.overscanFit
1199  qaStdev = float("NaN")
1200  else:
1201  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1202  afwMath.MEDIAN | afwMath.STDEVCLIP)
1203  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1204  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1205 
1206  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1207  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1208  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1209  amp.getName(), qaMedian, qaStdev)
1210  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1211  else:
1212  if badAmp:
1213  self.log.warn("Amplifier %s is bad.", amp.getName())
1214  overscanResults = None
1215 
1216  overscans.append(overscanResults if overscanResults is not None else None)
1217  else:
1218  self.log.info("Skipped OSCAN for %s.", amp.getName())
1219 
1220  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1221  self.log.info("Applying crosstalk correction.")
1222  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1223  self.debugView(ccdExposure, "doCrosstalk")
1224 
1225  if self.config.doAssembleCcd:
1226  self.log.info("Assembling CCD from amplifiers.")
1227  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1228 
1229  if self.config.expectWcs and not ccdExposure.getWcs():
1230  self.log.warn("No WCS found in input exposure.")
1231  self.debugView(ccdExposure, "doAssembleCcd")
1232 
1233  ossThumb = None
1234  if self.config.qa.doThumbnailOss:
1235  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1236 
1237  if self.config.doBias:
1238  self.log.info("Applying bias correction.")
1239  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1240  trimToFit=self.config.doTrimToMatchCalib)
1241  self.debugView(ccdExposure, "doBias")
1242 
1243  if self.config.doVariance:
1244  for amp, overscanResults in zip(ccd, overscans):
1245  if ccdExposure.getBBox().contains(amp.getBBox()):
1246  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1247  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1248  if overscanResults is not None:
1249  self.updateVariance(ampExposure, amp,
1250  overscanImage=overscanResults.overscanImage)
1251  else:
1252  self.updateVariance(ampExposure, amp,
1253  overscanImage=None)
1254  if self.config.qa is not None and self.config.qa.saveStats is True:
1255  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1256  afwMath.MEDIAN | afwMath.STDEVCLIP)
1257  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1258  qaStats.getValue(afwMath.MEDIAN))
1259  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1260  qaStats.getValue(afwMath.STDEVCLIP))
1261  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1262  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1263  qaStats.getValue(afwMath.STDEVCLIP))
1264 
1265  if self.doLinearize(ccd):
1266  self.log.info("Applying linearizer.")
1267  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1268 
1269  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1270  self.log.info("Applying crosstalk correction.")
1271  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1272  self.debugView(ccdExposure, "doCrosstalk")
1273 
1274  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1275  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1276  if self.config.doDefect:
1277  self.log.info("Masking defects.")
1278  self.maskDefect(ccdExposure, defects)
1279 
1280  if self.config.numEdgeSuspect > 0:
1281  self.log.info("Masking edges as SUSPECT.")
1282  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1283  maskPlane="SUSPECT")
1284 
1285  if self.config.doNanMasking:
1286  self.log.info("Masking NAN value pixels.")
1287  self.maskNan(ccdExposure)
1288 
1289  if self.config.doWidenSaturationTrails:
1290  self.log.info("Widening saturation trails.")
1291  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1292 
1293  if self.config.doCameraSpecificMasking:
1294  self.log.info("Masking regions for camera specific reasons.")
1295  self.masking.run(ccdExposure)
1296 
1297  if self.config.doBrighterFatter:
1298  # We need to apply flats and darks before we can interpolate, and we
1299  # need to interpolate before we do B-F, but we do B-F without the
1300  # flats and darks applied so we can work in units of electrons or holes.
1301  # This context manager applies and then removes the darks and flats.
1302  #
1303  # We also do not want to interpolate values here, so operate on temporary
1304  # images so we can apply only the BF-correction and roll back the
1305  # interpolation.
1306  interpExp = ccdExposure.clone()
1307  with self.flatContext(interpExp, flat, dark):
1308  isrFunctions.interpolateFromMask(
1309  maskedImage=interpExp.getMaskedImage(),
1310  fwhm=self.config.fwhm,
1311  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1312  maskNameList=self.config.maskListToInterpolate
1313  )
1314  bfExp = interpExp.clone()
1315 
1316  self.log.info("Applying brighter fatter correction.")
1317  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1318  self.config.brighterFatterMaxIter,
1319  self.config.brighterFatterThreshold,
1320  self.config.brighterFatterApplyGain
1321  )
1322  if bfResults[1] == self.config.brighterFatterMaxIter:
1323  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1324  bfResults[0])
1325  else:
1326  self.log.info("Finished brighter fatter correction in %d iterations.",
1327  bfResults[1])
1328  image = ccdExposure.getMaskedImage().getImage()
1329  bfCorr = bfExp.getMaskedImage().getImage()
1330  bfCorr -= interpExp.getMaskedImage().getImage()
1331  image += bfCorr
1332 
1333  # Applying the brighter-fatter correction applies a
1334  # convolution to the science image. At the edges this
1335  # convolution may not have sufficient valid pixels to
1336  # produce a valid correction. Mark pixels within the size
1337  # of the brighter-fatter kernel as EDGE to warn of this
1338  # fact.
1339  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1340  maskPlane="EDGE")
1341  self.log.warn("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1342 
1343  self.debugView(ccdExposure, "doBrighterFatter")
1344 
1345  if self.config.doDark:
1346  self.log.info("Applying dark correction.")
1347  self.darkCorrection(ccdExposure, dark)
1348  self.debugView(ccdExposure, "doDark")
1349 
1350  if self.config.doFringe and not self.config.fringeAfterFlat:
1351  self.log.info("Applying fringe correction before flat.")
1352  self.fringe.run(ccdExposure, **fringes.getDict())
1353  self.debugView(ccdExposure, "doFringe")
1354 
1355  if self.config.doStrayLight:
1356  if strayLightData is not None:
1357  self.log.info("Applying stray light correction.")
1358  self.strayLight.run(ccdExposure, strayLightData)
1359  self.debugView(ccdExposure, "doStrayLight")
1360  else:
1361  self.log.debug("Skipping stray light correction: no data found for this image.")
1362 
1363  if self.config.doFlat:
1364  self.log.info("Applying flat correction.")
1365  self.flatCorrection(ccdExposure, flat)
1366  self.debugView(ccdExposure, "doFlat")
1367 
1368  if self.config.doApplyGains:
1369  self.log.info("Applying gain correction instead of flat.")
1370  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1371 
1372  if self.config.doFringe and self.config.fringeAfterFlat:
1373  self.log.info("Applying fringe correction after flat.")
1374  self.fringe.run(ccdExposure, **fringes.getDict())
1375 
1376  if self.config.doVignette:
1377  self.log.info("Constructing Vignette polygon.")
1378  self.vignettePolygon = self.vignette.run(ccdExposure)
1379 
1380  if self.config.vignette.doWriteVignettePolygon:
1381  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1382 
1383  if self.config.doAttachTransmissionCurve:
1384  self.log.info("Adding transmission curves.")
1385  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1386  filterTransmission=filterTransmission,
1387  sensorTransmission=sensorTransmission,
1388  atmosphereTransmission=atmosphereTransmission)
1389 
1390  if self.config.doAddDistortionModel:
1391  self.log.info("Adding a distortion model to the WCS.")
1392  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1393 
1394  flattenedThumb = None
1395  if self.config.qa.doThumbnailFlattened:
1396  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1397 
1398  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1399  self.log.info("Performing illumination correction.")
1400  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1401  illumMaskedImage, illumScale=self.config.illumScale,
1402  trimToFit=self.config.doTrimToMatchCalib)
1403 
1404  preInterpExp = None
1405  if self.config.doSaveInterpPixels:
1406  preInterpExp = ccdExposure.clone()
1407 
1408  # Reset and interpolate bad pixels.
1409  #
1410  # Large contiguous bad regions (which should have the BAD mask
1411  # bit set) should have their values set to the image median.
1412  # This group should include defects and bad amplifiers. As the
1413  # area covered by these defects are large, there's little
1414  # reason to expect that interpolation would provide a more
1415  # useful value.
1416  #
1417  # Smaller defects can be safely interpolated after the larger
1418  # regions have had their pixel values reset. This ensures
1419  # that the remaining defects adjacent to bad amplifiers (as an
1420  # example) do not attempt to interpolate extreme values.
1421  if self.config.doSetBadRegions:
1422  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1423  if badPixelCount > 0:
1424  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1425 
1426  if self.config.doInterpolate:
1427  self.log.info("Interpolating masked pixels.")
1428  isrFunctions.interpolateFromMask(
1429  maskedImage=ccdExposure.getMaskedImage(),
1430  fwhm=self.config.fwhm,
1431  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1432  maskNameList=list(self.config.maskListToInterpolate)
1433  )
1434 
1435  self.roughZeroPoint(ccdExposure)
1436 
1437  if self.config.doMeasureBackground:
1438  self.log.info("Measuring background level.")
1439  self.measureBackground(ccdExposure, self.config.qa)
1440 
1441  if self.config.qa is not None and self.config.qa.saveStats is True:
1442  for amp in ccd:
1443  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1444  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1445  afwMath.MEDIAN | afwMath.STDEVCLIP)
1446  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1447  qaStats.getValue(afwMath.MEDIAN))
1448  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1449  qaStats.getValue(afwMath.STDEVCLIP))
1450  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1451  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1452  qaStats.getValue(afwMath.STDEVCLIP))
1453 
1454  self.debugView(ccdExposure, "postISRCCD")
1455 
1456  return pipeBase.Struct(
1457  exposure=ccdExposure,
1458  ossThumb=ossThumb,
1459  flattenedThumb=flattenedThumb,
1460 
1461  preInterpolatedExposure=preInterpExp,
1462  outputExposure=ccdExposure,
1463  outputOssThumbnail=ossThumb,
1464  outputFlattenedThumbnail=flattenedThumb,
1465  )
1466 
1467  @pipeBase.timeMethod
1468  def runDataRef(self, sensorRef):
1469  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1470 
1471  This method contains the `CmdLineTask` interface to the ISR
1472  processing. All IO is handled here, freeing the `run()` method
1473  to manage only pixel-level calculations. The steps performed
1474  are:
1475  - Read in necessary detrending/isr/calibration data.
1476  - Process raw exposure in `run()`.
1477  - Persist the ISR-corrected exposure as "postISRCCD" if
1478  config.doWrite=True.
1479 
1480  Parameters
1481  ----------
1482  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1483  DataRef of the detector data to be processed
1484 
1485  Returns
1486  -------
1487  result : `lsst.pipe.base.Struct`
1488  Result struct with component:
1489  - ``exposure`` : `afw.image.Exposure`
1490  The fully ISR corrected exposure.
1491 
1492  Raises
1493  ------
1494  RuntimeError
1495  Raised if a configuration option is set to True, but the
1496  required calibration data does not exist.
1497 
1498  """
1499  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1500 
1501  ccdExposure = sensorRef.get(self.config.datasetType)
1502 
1503  camera = sensorRef.get("camera")
1504  if camera is None and self.config.doAddDistortionModel:
1505  raise RuntimeError("config.doAddDistortionModel is True "
1506  "but could not get a camera from the butler.")
1507  isrData = self.readIsrData(sensorRef, ccdExposure)
1508 
1509  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1510 
1511  if self.config.doWrite:
1512  sensorRef.put(result.exposure, "postISRCCD")
1513  if result.preInterpolatedExposure is not None:
1514  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1515  if result.ossThumb is not None:
1516  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1517  if result.flattenedThumb is not None:
1518  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1519 
1520  return result
1521 
1522  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1523  """!Retrieve a calibration dataset for removing instrument signature.
1524 
1525  Parameters
1526  ----------
1527 
1528  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1529  DataRef of the detector data to find calibration datasets
1530  for.
1531  datasetType : `str`
1532  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1533  immediate : `Bool`
1534  If True, disable butler proxies to enable error handling
1535  within this routine.
1536 
1537  Returns
1538  -------
1539  exposure : `lsst.afw.image.Exposure`
1540  Requested calibration frame.
1541 
1542  Raises
1543  ------
1544  RuntimeError
1545  Raised if no matching calibration frame can be found.
1546  """
1547  try:
1548  exp = dataRef.get(datasetType, immediate=immediate)
1549  except Exception as exc1:
1550  if not self.config.fallbackFilterName:
1551  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1552  try:
1553  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1554  except Exception as exc2:
1555  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1556  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1557  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1558 
1559  if self.config.doAssembleIsrExposures:
1560  exp = self.assembleCcd.assembleCcd(exp)
1561  return exp
1562 
1563  def ensureExposure(self, inputExp, camera, detectorNum):
1564  """Ensure that the data returned by Butler is a fully constructed exposure.
1565 
1566  ISR requires exposure-level image data for historical reasons, so if we did
1567  not recieve that from Butler, construct it from what we have, modifying the
1568  input in place.
1569 
1570  Parameters
1571  ----------
1572  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1573  `lsst.afw.image.ImageF`
1574  The input data structure obtained from Butler.
1575  camera : `lsst.afw.cameraGeom.camera`
1576  The camera associated with the image. Used to find the appropriate
1577  detector.
1578  detectorNum : `int`
1579  The detector this exposure should match.
1580 
1581  Returns
1582  -------
1583  inputExp : `lsst.afw.image.Exposure`
1584  The re-constructed exposure, with appropriate detector parameters.
1585 
1586  Raises
1587  ------
1588  TypeError
1589  Raised if the input data cannot be used to construct an exposure.
1590  """
1591  if isinstance(inputExp, afwImage.DecoratedImageU):
1592  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1593  elif isinstance(inputExp, afwImage.ImageF):
1594  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1595  elif isinstance(inputExp, afwImage.MaskedImageF):
1596  inputExp = afwImage.makeExposure(inputExp)
1597  elif isinstance(inputExp, afwImage.Exposure):
1598  pass
1599  elif inputExp is None:
1600  # Assume this will be caught by the setup if it is a problem.
1601  return inputExp
1602  else:
1603  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1604  (type(inputExp), ))
1605 
1606  if inputExp.getDetector() is None:
1607  inputExp.setDetector(camera[detectorNum])
1608 
1609  return inputExp
1610 
1611  def convertIntToFloat(self, exposure):
1612  """Convert exposure image from uint16 to float.
1613 
1614  If the exposure does not need to be converted, the input is
1615  immediately returned. For exposures that are converted to use
1616  floating point pixels, the variance is set to unity and the
1617  mask to zero.
1618 
1619  Parameters
1620  ----------
1621  exposure : `lsst.afw.image.Exposure`
1622  The raw exposure to be converted.
1623 
1624  Returns
1625  -------
1626  newexposure : `lsst.afw.image.Exposure`
1627  The input ``exposure``, converted to floating point pixels.
1628 
1629  Raises
1630  ------
1631  RuntimeError
1632  Raised if the exposure type cannot be converted to float.
1633 
1634  """
1635  if isinstance(exposure, afwImage.ExposureF):
1636  # Nothing to be done
1637  self.log.debug("Exposure already of type float.")
1638  return exposure
1639  if not hasattr(exposure, "convertF"):
1640  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1641 
1642  newexposure = exposure.convertF()
1643  newexposure.variance[:] = 1
1644  newexposure.mask[:] = 0x0
1645 
1646  return newexposure
1647 
1648  def maskAmplifier(self, ccdExposure, amp, defects):
1649  """Identify bad amplifiers, saturated and suspect pixels.
1650 
1651  Parameters
1652  ----------
1653  ccdExposure : `lsst.afw.image.Exposure`
1654  Input exposure to be masked.
1655  amp : `lsst.afw.table.AmpInfoCatalog`
1656  Catalog of parameters defining the amplifier on this
1657  exposure to mask.
1658  defects : `lsst.meas.algorithms.Defects`
1659  List of defects. Used to determine if the entire
1660  amplifier is bad.
1661 
1662  Returns
1663  -------
1664  badAmp : `Bool`
1665  If this is true, the entire amplifier area is covered by
1666  defects and unusable.
1667 
1668  """
1669  maskedImage = ccdExposure.getMaskedImage()
1670 
1671  badAmp = False
1672 
1673  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1674  # comparison with current defects definition.
1675  if defects is not None:
1676  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1677 
1678  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1679  # association with pixels in current ccdExposure).
1680  if badAmp:
1681  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1682  afwImage.PARENT)
1683  maskView = dataView.getMask()
1684  maskView |= maskView.getPlaneBitMask("BAD")
1685  del maskView
1686  return badAmp
1687 
1688  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1689  # Saturation and suspect pixels can be masked now, though.
1690  limits = dict()
1691  if self.config.doSaturation and not badAmp:
1692  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1693  if self.config.doSuspect and not badAmp:
1694  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1695  if math.isfinite(self.config.saturation):
1696  limits.update({self.config.saturatedMaskName: self.config.saturation})
1697 
1698  for maskName, maskThreshold in limits.items():
1699  if not math.isnan(maskThreshold):
1700  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1701  isrFunctions.makeThresholdMask(
1702  maskedImage=dataView,
1703  threshold=maskThreshold,
1704  growFootprints=0,
1705  maskName=maskName
1706  )
1707 
1708  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1709  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1710  afwImage.PARENT)
1711  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1712  self.config.suspectMaskName])
1713  if numpy.all(maskView.getArray() & maskVal > 0):
1714  badAmp = True
1715  maskView |= maskView.getPlaneBitMask("BAD")
1716 
1717  return badAmp
1718 
1719  def overscanCorrection(self, ccdExposure, amp):
1720  """Apply overscan correction in place.
1721 
1722  This method does initial pixel rejection of the overscan
1723  region. The overscan can also be optionally segmented to
1724  allow for discontinuous overscan responses to be fit
1725  separately. The actual overscan subtraction is performed by
1726  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1727  which is called here after the amplifier is preprocessed.
1728 
1729  Parameters
1730  ----------
1731  ccdExposure : `lsst.afw.image.Exposure`
1732  Exposure to have overscan correction performed.
1733  amp : `lsst.afw.table.AmpInfoCatalog`
1734  The amplifier to consider while correcting the overscan.
1735 
1736  Returns
1737  -------
1738  overscanResults : `lsst.pipe.base.Struct`
1739  Result struct with components:
1740  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1741  Value or fit subtracted from the amplifier image data.
1742  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1743  Value or fit subtracted from the overscan image data.
1744  - ``overscanImage`` : `lsst.afw.image.Image`
1745  Image of the overscan region with the overscan
1746  correction applied. This quantity is used to estimate
1747  the amplifier read noise empirically.
1748 
1749  Raises
1750  ------
1751  RuntimeError
1752  Raised if the ``amp`` does not contain raw pixel information.
1753 
1754  See Also
1755  --------
1756  lsst.ip.isr.isrFunctions.overscanCorrection
1757  """
1758  if not amp.getHasRawInfo():
1759  raise RuntimeError("This method must be executed on an amp with raw information.")
1760 
1761  if amp.getRawHorizontalOverscanBBox().isEmpty():
1762  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1763  return None
1764 
1765  statControl = afwMath.StatisticsControl()
1766  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1767 
1768  # Determine the bounding boxes
1769  dataBBox = amp.getRawDataBBox()
1770  oscanBBox = amp.getRawHorizontalOverscanBBox()
1771  dx0 = 0
1772  dx1 = 0
1773 
1774  prescanBBox = amp.getRawPrescanBBox()
1775  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1776  dx0 += self.config.overscanNumLeadingColumnsToSkip
1777  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1778  else:
1779  dx0 += self.config.overscanNumTrailingColumnsToSkip
1780  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1781 
1782  # Determine if we need to work on subregions of the amplifier and overscan.
1783  imageBBoxes = []
1784  overscanBBoxes = []
1785 
1786  if ((self.config.overscanBiasJump and
1787  self.config.overscanBiasJumpLocation) and
1788  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1789  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1790  self.config.overscanBiasJumpDevices)):
1791  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1792  yLower = self.config.overscanBiasJumpLocation
1793  yUpper = dataBBox.getHeight() - yLower
1794  else:
1795  yUpper = self.config.overscanBiasJumpLocation
1796  yLower = dataBBox.getHeight() - yUpper
1797 
1798  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1799  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1800  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1801  lsst.geom.Extent2I(dx0, 0),
1802  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1803  yLower)))
1804 
1805  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1806  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1807  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1808  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1809  yUpper)))
1810  else:
1811  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1812  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1813  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1814  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1815  oscanBBox.getHeight())))
1816 
1817  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1818  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1819  ampImage = ccdExposure.maskedImage[imageBBox]
1820  overscanImage = ccdExposure.maskedImage[overscanBBox]
1821 
1822  overscanArray = overscanImage.image.array
1823  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1824  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1825  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1826 
1827  statControl = afwMath.StatisticsControl()
1828  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1829 
1830  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1831  overscanImage=overscanImage,
1832  fitType=self.config.overscanFitType,
1833  order=self.config.overscanOrder,
1834  collapseRej=self.config.overscanNumSigmaClip,
1835  statControl=statControl,
1836  overscanIsInt=self.config.overscanIsInt
1837  )
1838 
1839  # Measure average overscan levels and record them in the metadata.
1840  levelStat = afwMath.MEDIAN
1841  sigmaStat = afwMath.STDEVCLIP
1842 
1843  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1844  self.config.qa.flatness.nIter)
1845  metadata = ccdExposure.getMetadata()
1846  ampNum = amp.getName()
1847  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1848  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1849  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1850  else:
1851  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1852  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1853  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1854 
1855  return overscanResults
1856 
1857  def updateVariance(self, ampExposure, amp, overscanImage=None):
1858  """Set the variance plane using the amplifier gain and read noise
1859 
1860  The read noise is calculated from the ``overscanImage`` if the
1861  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1862  the value from the amplifier data is used.
1863 
1864  Parameters
1865  ----------
1866  ampExposure : `lsst.afw.image.Exposure`
1867  Exposure to process.
1868  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1869  Amplifier detector data.
1870  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1871  Image of overscan, required only for empirical read noise.
1872 
1873  See also
1874  --------
1875  lsst.ip.isr.isrFunctions.updateVariance
1876  """
1877  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1878  gain = amp.getGain()
1879 
1880  if math.isnan(gain):
1881  gain = 1.0
1882  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1883  elif gain <= 0:
1884  patchedGain = 1.0
1885  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1886  amp.getName(), gain, patchedGain)
1887  gain = patchedGain
1888 
1889  if self.config.doEmpiricalReadNoise and overscanImage is None:
1890  self.log.info("Overscan is none for EmpiricalReadNoise.")
1891 
1892  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1893  stats = afwMath.StatisticsControl()
1894  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1895  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1896  self.log.info("Calculated empirical read noise for amp %s: %f.",
1897  amp.getName(), readNoise)
1898  else:
1899  readNoise = amp.getReadNoise()
1900 
1901  isrFunctions.updateVariance(
1902  maskedImage=ampExposure.getMaskedImage(),
1903  gain=gain,
1904  readNoise=readNoise,
1905  )
1906 
1907  def darkCorrection(self, exposure, darkExposure, invert=False):
1908  """!Apply dark correction in place.
1909 
1910  Parameters
1911  ----------
1912  exposure : `lsst.afw.image.Exposure`
1913  Exposure to process.
1914  darkExposure : `lsst.afw.image.Exposure`
1915  Dark exposure of the same size as ``exposure``.
1916  invert : `Bool`, optional
1917  If True, re-add the dark to an already corrected image.
1918 
1919  Raises
1920  ------
1921  RuntimeError
1922  Raised if either ``exposure`` or ``darkExposure`` do not
1923  have their dark time defined.
1924 
1925  See Also
1926  --------
1927  lsst.ip.isr.isrFunctions.darkCorrection
1928  """
1929  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1930  if math.isnan(expScale):
1931  raise RuntimeError("Exposure darktime is NAN.")
1932  if darkExposure.getInfo().getVisitInfo() is not None:
1933  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1934  else:
1935  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1936  # so getDarkTime() does not exist.
1937  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1938  darkScale = 1.0
1939 
1940  if math.isnan(darkScale):
1941  raise RuntimeError("Dark calib darktime is NAN.")
1942  isrFunctions.darkCorrection(
1943  maskedImage=exposure.getMaskedImage(),
1944  darkMaskedImage=darkExposure.getMaskedImage(),
1945  expScale=expScale,
1946  darkScale=darkScale,
1947  invert=invert,
1948  trimToFit=self.config.doTrimToMatchCalib
1949  )
1950 
1951  def doLinearize(self, detector):
1952  """!Check if linearization is needed for the detector cameraGeom.
1953 
1954  Checks config.doLinearize and the linearity type of the first
1955  amplifier.
1956 
1957  Parameters
1958  ----------
1959  detector : `lsst.afw.cameraGeom.Detector`
1960  Detector to get linearity type from.
1961 
1962  Returns
1963  -------
1964  doLinearize : `Bool`
1965  If True, linearization should be performed.
1966  """
1967  return self.config.doLinearize and \
1968  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1969 
1970  def flatCorrection(self, exposure, flatExposure, invert=False):
1971  """!Apply flat correction in place.
1972 
1973  Parameters
1974  ----------
1975  exposure : `lsst.afw.image.Exposure`
1976  Exposure to process.
1977  flatExposure : `lsst.afw.image.Exposure`
1978  Flat exposure of the same size as ``exposure``.
1979  invert : `Bool`, optional
1980  If True, unflatten an already flattened image.
1981 
1982  See Also
1983  --------
1984  lsst.ip.isr.isrFunctions.flatCorrection
1985  """
1986  isrFunctions.flatCorrection(
1987  maskedImage=exposure.getMaskedImage(),
1988  flatMaskedImage=flatExposure.getMaskedImage(),
1989  scalingType=self.config.flatScalingType,
1990  userScale=self.config.flatUserScale,
1991  invert=invert,
1992  trimToFit=self.config.doTrimToMatchCalib
1993  )
1994 
1995  def saturationDetection(self, exposure, amp):
1996  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1997 
1998  Parameters
1999  ----------
2000  exposure : `lsst.afw.image.Exposure`
2001  Exposure to process. Only the amplifier DataSec is processed.
2002  amp : `lsst.afw.table.AmpInfoCatalog`
2003  Amplifier detector data.
2004 
2005  See Also
2006  --------
2007  lsst.ip.isr.isrFunctions.makeThresholdMask
2008  """
2009  if not math.isnan(amp.getSaturation()):
2010  maskedImage = exposure.getMaskedImage()
2011  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2012  isrFunctions.makeThresholdMask(
2013  maskedImage=dataView,
2014  threshold=amp.getSaturation(),
2015  growFootprints=0,
2016  maskName=self.config.saturatedMaskName,
2017  )
2018 
2019  def saturationInterpolation(self, exposure):
2020  """!Interpolate over saturated pixels, in place.
2021 
2022  This method should be called after `saturationDetection`, to
2023  ensure that the saturated pixels have been identified in the
2024  SAT mask. It should also be called after `assembleCcd`, since
2025  saturated regions may cross amplifier boundaries.
2026 
2027  Parameters
2028  ----------
2029  exposure : `lsst.afw.image.Exposure`
2030  Exposure to process.
2031 
2032  See Also
2033  --------
2034  lsst.ip.isr.isrTask.saturationDetection
2035  lsst.ip.isr.isrFunctions.interpolateFromMask
2036  """
2037  isrFunctions.interpolateFromMask(
2038  maskedImage=exposure.getMaskedImage(),
2039  fwhm=self.config.fwhm,
2040  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2041  maskNameList=list(self.config.saturatedMaskName),
2042  )
2043 
2044  def suspectDetection(self, exposure, amp):
2045  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2046 
2047  Parameters
2048  ----------
2049  exposure : `lsst.afw.image.Exposure`
2050  Exposure to process. Only the amplifier DataSec is processed.
2051  amp : `lsst.afw.table.AmpInfoCatalog`
2052  Amplifier detector data.
2053 
2054  See Also
2055  --------
2056  lsst.ip.isr.isrFunctions.makeThresholdMask
2057 
2058  Notes
2059  -----
2060  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2061  This is intended to indicate pixels that may be affected by unknown systematics;
2062  for example if non-linearity corrections above a certain level are unstable
2063  then that would be a useful value for suspectLevel. A value of `nan` indicates
2064  that no such level exists and no pixels are to be masked as suspicious.
2065  """
2066  suspectLevel = amp.getSuspectLevel()
2067  if math.isnan(suspectLevel):
2068  return
2069 
2070  maskedImage = exposure.getMaskedImage()
2071  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2072  isrFunctions.makeThresholdMask(
2073  maskedImage=dataView,
2074  threshold=suspectLevel,
2075  growFootprints=0,
2076  maskName=self.config.suspectMaskName,
2077  )
2078 
2079  def maskDefect(self, exposure, defectBaseList):
2080  """!Mask defects using mask plane "BAD", in place.
2081 
2082  Parameters
2083  ----------
2084  exposure : `lsst.afw.image.Exposure`
2085  Exposure to process.
2086  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2087  `lsst.afw.image.DefectBase`.
2088  List of defects to mask.
2089 
2090  Notes
2091  -----
2092  Call this after CCD assembly, since defects may cross amplifier boundaries.
2093  """
2094  maskedImage = exposure.getMaskedImage()
2095  if not isinstance(defectBaseList, Defects):
2096  # Promotes DefectBase to Defect
2097  defectList = Defects(defectBaseList)
2098  else:
2099  defectList = defectBaseList
2100  defectList.maskPixels(maskedImage, maskName="BAD")
2101 
2102  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2103  """!Mask edge pixels with applicable mask plane.
2104 
2105  Parameters
2106  ----------
2107  exposure : `lsst.afw.image.Exposure`
2108  Exposure to process.
2109  numEdgePixels : `int`, optional
2110  Number of edge pixels to mask.
2111  maskPlane : `str`, optional
2112  Mask plane name to use.
2113  """
2114  maskedImage = exposure.getMaskedImage()
2115  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2116 
2117  if numEdgePixels > 0:
2118  goodBBox = maskedImage.getBBox()
2119  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2120  goodBBox.grow(-numEdgePixels)
2121  # Mask pixels outside goodBBox
2122  SourceDetectionTask.setEdgeBits(
2123  maskedImage,
2124  goodBBox,
2125  maskBitMask
2126  )
2127 
2128  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2129  """Mask and interpolate defects using mask plane "BAD", in place.
2130 
2131  Parameters
2132  ----------
2133  exposure : `lsst.afw.image.Exposure`
2134  Exposure to process.
2135  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2136  `lsst.afw.image.DefectBase`.
2137  List of defects to mask and interpolate.
2138 
2139  See Also
2140  --------
2141  lsst.ip.isr.isrTask.maskDefect()
2142  """
2143  self.maskDefect(exposure, defectBaseList)
2144  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2145  maskPlane="SUSPECT")
2146  isrFunctions.interpolateFromMask(
2147  maskedImage=exposure.getMaskedImage(),
2148  fwhm=self.config.fwhm,
2149  growSaturatedFootprints=0,
2150  maskNameList=["BAD"],
2151  )
2152 
2153  def maskNan(self, exposure):
2154  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2155 
2156  Parameters
2157  ----------
2158  exposure : `lsst.afw.image.Exposure`
2159  Exposure to process.
2160 
2161  Notes
2162  -----
2163  We mask over all NaNs, including those that are masked with
2164  other bits (because those may or may not be interpolated over
2165  later, and we want to remove all NaNs). Despite this
2166  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2167  the historical name.
2168  """
2169  maskedImage = exposure.getMaskedImage()
2170 
2171  # Find and mask NaNs
2172  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2173  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2174  numNans = maskNans(maskedImage, maskVal)
2175  self.metadata.set("NUMNANS", numNans)
2176  if numNans > 0:
2177  self.log.warn("There were %d unmasked NaNs.", numNans)
2178 
2179  def maskAndInterpolateNan(self, exposure):
2180  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2181 
2182  Parameters
2183  ----------
2184  exposure : `lsst.afw.image.Exposure`
2185  Exposure to process.
2186 
2187  See Also
2188  --------
2189  lsst.ip.isr.isrTask.maskNan()
2190  """
2191  self.maskNan(exposure)
2192  isrFunctions.interpolateFromMask(
2193  maskedImage=exposure.getMaskedImage(),
2194  fwhm=self.config.fwhm,
2195  growSaturatedFootprints=0,
2196  maskNameList=["UNMASKEDNAN"],
2197  )
2198 
2199  def measureBackground(self, exposure, IsrQaConfig=None):
2200  """Measure the image background in subgrids, for quality control purposes.
2201 
2202  Parameters
2203  ----------
2204  exposure : `lsst.afw.image.Exposure`
2205  Exposure to process.
2206  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2207  Configuration object containing parameters on which background
2208  statistics and subgrids to use.
2209  """
2210  if IsrQaConfig is not None:
2211  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2212  IsrQaConfig.flatness.nIter)
2213  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2214  statsControl.setAndMask(maskVal)
2215  maskedImage = exposure.getMaskedImage()
2216  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2217  skyLevel = stats.getValue(afwMath.MEDIAN)
2218  skySigma = stats.getValue(afwMath.STDEVCLIP)
2219  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2220  metadata = exposure.getMetadata()
2221  metadata.set('SKYLEVEL', skyLevel)
2222  metadata.set('SKYSIGMA', skySigma)
2223 
2224  # calcluating flatlevel over the subgrids
2225  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2226  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2227  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2228  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2229  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2230  skyLevels = numpy.zeros((nX, nY))
2231 
2232  for j in range(nY):
2233  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2234  for i in range(nX):
2235  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2236 
2237  xLLC = xc - meshXHalf
2238  yLLC = yc - meshYHalf
2239  xURC = xc + meshXHalf - 1
2240  yURC = yc + meshYHalf - 1
2241 
2242  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2243  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2244 
2245  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2246 
2247  good = numpy.where(numpy.isfinite(skyLevels))
2248  skyMedian = numpy.median(skyLevels[good])
2249  flatness = (skyLevels[good] - skyMedian) / skyMedian
2250  flatness_rms = numpy.std(flatness)
2251  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2252 
2253  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2254  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2255  nX, nY, flatness_pp, flatness_rms)
2256 
2257  metadata.set('FLATNESS_PP', float(flatness_pp))
2258  metadata.set('FLATNESS_RMS', float(flatness_rms))
2259  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2260  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2261  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2262 
2263  def roughZeroPoint(self, exposure):
2264  """Set an approximate magnitude zero point for the exposure.
2265 
2266  Parameters
2267  ----------
2268  exposure : `lsst.afw.image.Exposure`
2269  Exposure to process.
2270  """
2271  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2272  if filterName in self.config.fluxMag0T1:
2273  fluxMag0 = self.config.fluxMag0T1[filterName]
2274  else:
2275  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2276  fluxMag0 = self.config.defaultFluxMag0T1
2277 
2278  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2279  if not expTime > 0: # handle NaN as well as <= 0
2280  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2281  return
2282 
2283  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2284  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2285 
2286  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2287  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2288 
2289  Parameters
2290  ----------
2291  ccdExposure : `lsst.afw.image.Exposure`
2292  Exposure to process.
2293  fpPolygon : `lsst.afw.geom.Polygon`
2294  Polygon in focal plane coordinates.
2295  """
2296  # Get ccd corners in focal plane coordinates
2297  ccd = ccdExposure.getDetector()
2298  fpCorners = ccd.getCorners(FOCAL_PLANE)
2299  ccdPolygon = Polygon(fpCorners)
2300 
2301  # Get intersection of ccd corners with fpPolygon
2302  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2303 
2304  # Transform back to pixel positions and build new polygon
2305  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2306  validPolygon = Polygon(ccdPoints)
2307  ccdExposure.getInfo().setValidPolygon(validPolygon)
2308 
2309  @contextmanager
2310  def flatContext(self, exp, flat, dark=None):
2311  """Context manager that applies and removes flats and darks,
2312  if the task is configured to apply them.
2313 
2314  Parameters
2315  ----------
2316  exp : `lsst.afw.image.Exposure`
2317  Exposure to process.
2318  flat : `lsst.afw.image.Exposure`
2319  Flat exposure the same size as ``exp``.
2320  dark : `lsst.afw.image.Exposure`, optional
2321  Dark exposure the same size as ``exp``.
2322 
2323  Yields
2324  ------
2325  exp : `lsst.afw.image.Exposure`
2326  The flat and dark corrected exposure.
2327  """
2328  if self.config.doDark and dark is not None:
2329  self.darkCorrection(exp, dark)
2330  if self.config.doFlat:
2331  self.flatCorrection(exp, flat)
2332  try:
2333  yield exp
2334  finally:
2335  if self.config.doFlat:
2336  self.flatCorrection(exp, flat, invert=True)
2337  if self.config.doDark and dark is not None:
2338  self.darkCorrection(exp, dark, invert=True)
2339 
2340  def debugView(self, exposure, stepname):
2341  """Utility function to examine ISR exposure at different stages.
2342 
2343  Parameters
2344  ----------
2345  exposure : `lsst.afw.image.Exposure`
2346  Exposure to view.
2347  stepname : `str`
2348  State of processing to view.
2349  """
2350  frame = getDebugFrame(self._display, stepname)
2351  if frame:
2352  display = getDisplay(frame)
2353  display.scale('asinh', 'zscale')
2354  display.mtv(exposure)
2355  prompt = "Press Enter to continue [c]... "
2356  while True:
2357  ans = input(prompt).lower()
2358  if ans in ("", "c",):
2359  break
2360 
2361 
2362 class FakeAmp(object):
2363  """A Detector-like object that supports returning gain and saturation level
2364 
2365  This is used when the input exposure does not have a detector.
2366 
2367  Parameters
2368  ----------
2369  exposure : `lsst.afw.image.Exposure`
2370  Exposure to generate a fake amplifier for.
2371  config : `lsst.ip.isr.isrTaskConfig`
2372  Configuration to apply to the fake amplifier.
2373  """
2374 
2375  def __init__(self, exposure, config):
2376  self._bbox = exposure.getBBox(afwImage.LOCAL)
2378  self._gain = config.gain
2379  self._readNoise = config.readNoise
2380  self._saturation = config.saturation
2381 
2382  def getBBox(self):
2383  return self._bbox
2384 
2385  def getRawBBox(self):
2386  return self._bbox
2387 
2388  def getHasRawInfo(self):
2389  return True # but see getRawHorizontalOverscanBBox()
2390 
2392  return self._RawHorizontalOverscanBBox
2393 
2394  def getGain(self):
2395  return self._gain
2396 
2397  def getReadNoise(self):
2398  return self._readNoise
2399 
2400  def getSaturation(self):
2401  return self._saturation
2402 
2403  def getSuspectLevel(self):
2404  return float("NaN")
2405 
2406 
2407 class RunIsrConfig(pexConfig.Config):
2408  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2409 
2410 
2411 class RunIsrTask(pipeBase.CmdLineTask):
2412  """Task to wrap the default IsrTask to allow it to be retargeted.
2413 
2414  The standard IsrTask can be called directly from a command line
2415  program, but doing so removes the ability of the task to be
2416  retargeted. As most cameras override some set of the IsrTask
2417  methods, this would remove those data-specific methods in the
2418  output post-ISR images. This wrapping class fixes the issue,
2419  allowing identical post-ISR images to be generated by both the
2420  processCcd and isrTask code.
2421  """
2422  ConfigClass = RunIsrConfig
2423  _DefaultName = "runIsr"
2424 
2425  def __init__(self, *args, **kwargs):
2426  super().__init__(*args, **kwargs)
2427  self.makeSubtask("isr")
2428 
2429  def runDataRef(self, dataRef):
2430  """
2431  Parameters
2432  ----------
2433  dataRef : `lsst.daf.persistence.ButlerDataRef`
2434  data reference of the detector data to be processed
2435 
2436  Returns
2437  -------
2438  result : `pipeBase.Struct`
2439  Result struct with component:
2440 
2441  - exposure : `lsst.afw.image.Exposure`
2442  Post-ISR processed exposure.
2443  """
2444  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:764
def runDataRef(self, sensorRef)
Definition: isrTask.py:1468
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2199
def debugView(self, exposure, stepname)
Definition: isrTask.py:2340
def __init__(self, kwargs)
Definition: isrTask.py:754
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1563
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:866
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:825
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2102
def runDataRef(self, dataRef)
Definition: isrTask.py:2429
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1014
def __init__(self, args, kwargs)
Definition: isrTask.py:2425
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:816
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2263
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2128
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2391
def maskNan(self, exposure)
Definition: isrTask.py:2153
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:803
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2079
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1719
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1611
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1970
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:863
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1522
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1907
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1951
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2286
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1648
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2310
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1857
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2179
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2044
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2019
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1995
def __init__(self, exposure, config)
Definition: isrTask.py:2375