lsst.ip.isr  18.1.0-12-gc95f69a+5
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 
56 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
57 
58 
59 class IsrTaskConfig(pexConfig.Config):
60  """Configuration parameters for IsrTask.
61 
62  Items are grouped in the order in which they are executed by the task.
63  """
64  # General ISR configuration
65 
66  # gen3 options
67  isrName = pexConfig.Field(
68  dtype=str,
69  doc="Name of ISR",
70  default="ISR",
71  )
72 
73  # input datasets
74  ccdExposure = pipeBase.InputDatasetField(
75  doc="Input exposure to process",
76  name="raw",
77  scalar=True,
78  storageClass="Exposure",
79  dimensions=["instrument", "exposure", "detector"],
80  )
81  camera = pipeBase.InputDatasetField(
82  doc="Input camera to construct complete exposures.",
83  name="camera",
84  scalar=True,
85  storageClass="Camera",
86  dimensions=["instrument", "calibration_label"],
87  )
88  bias = pipeBase.InputDatasetField(
89  doc="Input bias calibration.",
90  name="bias",
91  scalar=True,
92  storageClass="ImageF",
93  dimensions=["instrument", "calibration_label", "detector"],
94  )
95  dark = pipeBase.InputDatasetField(
96  doc="Input dark calibration.",
97  name="dark",
98  scalar=True,
99  storageClass="ImageF",
100  dimensions=["instrument", "calibration_label", "detector"],
101  )
102  flat = pipeBase.InputDatasetField(
103  doc="Input flat calibration.",
104  name="flat",
105  scalar=True,
106  storageClass="MaskedImageF",
107  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
108  )
109  bfKernel = pipeBase.InputDatasetField(
110  doc="Input brighter-fatter kernel.",
111  name="bfKernel",
112  scalar=True,
113  storageClass="NumpyArray",
114  dimensions=["instrument", "calibration_label"],
115  )
116  defects = pipeBase.InputDatasetField(
117  doc="Input defect tables.",
118  name="defects",
119  scalar=True,
120  storageClass="DefectsList",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  opticsTransmission = pipeBase.InputDatasetField(
124  doc="Transmission curve due to the optics.",
125  name="transmission_optics",
126  scalar=True,
127  storageClass="TransmissionCurve",
128  dimensions=["instrument", "calibration_label"],
129  )
130  filterTransmission = pipeBase.InputDatasetField(
131  doc="Transmission curve due to the filter.",
132  name="transmission_filter",
133  scalar=True,
134  storageClass="TransmissionCurve",
135  dimensions=["instrument", "physical_filter", "calibration_label"],
136  )
137  sensorTransmission = pipeBase.InputDatasetField(
138  doc="Transmission curve due to the sensor.",
139  name="transmission_sensor",
140  scalar=True,
141  storageClass="TransmissionCurve",
142  dimensions=["instrument", "calibration_label", "detector"],
143  )
144  atmosphereTransmission = pipeBase.InputDatasetField(
145  doc="Transmission curve due to the atmosphere.",
146  name="transmission_atmosphere",
147  scalar=True,
148  storageClass="TransmissionCurve",
149  dimensions=["instrument"],
150  )
151  illumMaskedImage = pipeBase.InputDatasetField(
152  doc="Input illumination correction.",
153  name="illum",
154  scalar=True,
155  storageClass="MaskedImageF",
156  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
157  )
158 
159  # output datasets
160  outputExposure = pipeBase.OutputDatasetField(
161  doc="Output ISR processed exposure.",
162  name="postISRCCD",
163  scalar=True,
164  storageClass="ExposureF",
165  dimensions=["instrument", "visit", "detector"],
166  )
167  outputOssThumbnail = pipeBase.OutputDatasetField(
168  doc="Output Overscan-subtracted thumbnail image.",
169  name="OssThumb",
170  scalar=True,
171  storageClass="Thumbnail",
172  dimensions=["instrument", "visit", "detector"],
173  )
174  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
175  doc="Output flat-corrected thumbnail image.",
176  name="FlattenedThumb",
177  scalar=True,
178  storageClass="TextStorage",
179  dimensions=["instrument", "visit", "detector"],
180  )
181 
182  quantum = pipeBase.QuantumConfig(
183  dimensions=["visit", "detector", "instrument"],
184  )
185 
186 
187  datasetType = pexConfig.Field(
188  dtype=str,
189  doc="Dataset type for input data; users will typically leave this alone, "
190  "but camera-specific ISR tasks will override it",
191  default="raw",
192  )
193 
194  fallbackFilterName = pexConfig.Field(
195  dtype=str,
196  doc="Fallback default filter name for calibrations.",
197  optional=True
198  )
199  expectWcs = pexConfig.Field(
200  dtype=bool,
201  default=True,
202  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
203  )
204  fwhm = pexConfig.Field(
205  dtype=float,
206  doc="FWHM of PSF in arcseconds.",
207  default=1.0,
208  )
209  qa = pexConfig.ConfigField(
210  dtype=isrQa.IsrQaConfig,
211  doc="QA related configuration options.",
212  )
213 
214  # Image conversion configuration
215  doConvertIntToFloat = pexConfig.Field(
216  dtype=bool,
217  doc="Convert integer raw images to floating point values?",
218  default=True,
219  )
220 
221  # Saturated pixel handling.
222  doSaturation = pexConfig.Field(
223  dtype=bool,
224  doc="Mask saturated pixels? NB: this is totally independent of the"
225  " interpolation option - this is ONLY setting the bits in the mask."
226  " To have them interpolated make sure doSaturationInterpolation=True",
227  default=True,
228  )
229  saturatedMaskName = pexConfig.Field(
230  dtype=str,
231  doc="Name of mask plane to use in saturation detection and interpolation",
232  default="SAT",
233  )
234  saturation = pexConfig.Field(
235  dtype=float,
236  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
237  default=float("NaN"),
238  )
239  growSaturationFootprintSize = pexConfig.Field(
240  dtype=int,
241  doc="Number of pixels by which to grow the saturation footprints",
242  default=1,
243  )
244 
245  # Suspect pixel handling.
246  doSuspect = pexConfig.Field(
247  dtype=bool,
248  doc="Mask suspect pixels?",
249  default=True,
250  )
251  suspectMaskName = pexConfig.Field(
252  dtype=str,
253  doc="Name of mask plane to use for suspect pixels",
254  default="SUSPECT",
255  )
256  numEdgeSuspect = pexConfig.Field(
257  dtype=int,
258  doc="Number of edge pixels to be flagged as untrustworthy.",
259  default=0,
260  )
261 
262  # Initial masking options.
263  doSetBadRegions = pexConfig.Field(
264  dtype=bool,
265  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
266  default=True,
267  )
268  badStatistic = pexConfig.ChoiceField(
269  dtype=str,
270  doc="How to estimate the average value for BAD regions.",
271  default='MEANCLIP',
272  allowed={
273  "MEANCLIP": "Correct using the (clipped) mean of good data",
274  "MEDIAN": "Correct using the median of the good data",
275  },
276  )
277 
278  # Overscan subtraction configuration.
279  doOverscan = pexConfig.Field(
280  dtype=bool,
281  doc="Do overscan subtraction?",
282  default=True,
283  )
284  overscanFitType = pexConfig.ChoiceField(
285  dtype=str,
286  doc="The method for fitting the overscan bias level.",
287  default='MEDIAN',
288  allowed={
289  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
290  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
291  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
292  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
293  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
294  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
295  "MEAN": "Correct using the mean of the overscan region",
296  "MEANCLIP": "Correct using a clipped mean of the overscan region",
297  "MEDIAN": "Correct using the median of the overscan region",
298  },
299  )
300  overscanOrder = pexConfig.Field(
301  dtype=int,
302  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
303  "or number of spline knots if overscan fit type is a spline."),
304  default=1,
305  )
306  overscanNumSigmaClip = pexConfig.Field(
307  dtype=float,
308  doc="Rejection threshold (sigma) for collapsing overscan before fit",
309  default=3.0,
310  )
311  overscanIsInt = pexConfig.Field(
312  dtype=bool,
313  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
314  default=True,
315  )
316  overscanNumLeadingColumnsToSkip = pexConfig.Field(
317  dtype=int,
318  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
319  default=0,
320  )
321  overscanNumTrailingColumnsToSkip = pexConfig.Field(
322  dtype=int,
323  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
324  default=0,
325  )
326  overscanMaxDev = pexConfig.Field(
327  dtype=float,
328  doc="Maximum deviation from the median for overscan",
329  default=1000.0, check=lambda x: x > 0
330  )
331  overscanBiasJump = pexConfig.Field(
332  dtype=bool,
333  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
334  default=False,
335  )
336  overscanBiasJumpKeyword = pexConfig.Field(
337  dtype=str,
338  doc="Header keyword containing information about devices.",
339  default="NO_SUCH_KEY",
340  )
341  overscanBiasJumpDevices = pexConfig.ListField(
342  dtype=str,
343  doc="List of devices that need piecewise overscan correction.",
344  default=(),
345  )
346  overscanBiasJumpLocation = pexConfig.Field(
347  dtype=int,
348  doc="Location of bias jump along y-axis.",
349  default=0,
350  )
351 
352  # Amplifier to CCD assembly configuration
353  doAssembleCcd = pexConfig.Field(
354  dtype=bool,
355  default=True,
356  doc="Assemble amp-level exposures into a ccd-level exposure?"
357  )
358  assembleCcd = pexConfig.ConfigurableField(
359  target=AssembleCcdTask,
360  doc="CCD assembly task",
361  )
362 
363  # General calibration configuration.
364  doAssembleIsrExposures = pexConfig.Field(
365  dtype=bool,
366  default=False,
367  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
368  )
369  doTrimToMatchCalib = pexConfig.Field(
370  dtype=bool,
371  default=False,
372  doc="Trim raw data to match calibration bounding boxes?"
373  )
374 
375  # Bias subtraction.
376  doBias = pexConfig.Field(
377  dtype=bool,
378  doc="Apply bias frame correction?",
379  default=True,
380  )
381  biasDataProductName = pexConfig.Field(
382  dtype=str,
383  doc="Name of the bias data product",
384  default="bias",
385  )
386 
387  # Variance construction
388  doVariance = pexConfig.Field(
389  dtype=bool,
390  doc="Calculate variance?",
391  default=True
392  )
393  gain = pexConfig.Field(
394  dtype=float,
395  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
396  default=float("NaN"),
397  )
398  readNoise = pexConfig.Field(
399  dtype=float,
400  doc="The read noise to use if no Detector is present in the Exposure",
401  default=0.0,
402  )
403  doEmpiricalReadNoise = pexConfig.Field(
404  dtype=bool,
405  default=False,
406  doc="Calculate empirical read noise instead of value from AmpInfo data?"
407  )
408 
409  # Linearization.
410  doLinearize = pexConfig.Field(
411  dtype=bool,
412  doc="Correct for nonlinearity of the detector's response?",
413  default=True,
414  )
415 
416  # Crosstalk.
417  doCrosstalk = pexConfig.Field(
418  dtype=bool,
419  doc="Apply intra-CCD crosstalk correction?",
420  default=False,
421  )
422  doCrosstalkBeforeAssemble = pexConfig.Field(
423  dtype=bool,
424  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
425  default=False,
426  )
427  crosstalk = pexConfig.ConfigurableField(
428  target=CrosstalkTask,
429  doc="Intra-CCD crosstalk correction",
430  )
431 
432  # Masking options.
433  doDefect = pexConfig.Field(
434  dtype=bool,
435  doc="Apply correction for CCD defects, e.g. hot pixels?",
436  default=True,
437  )
438  doNanMasking = pexConfig.Field(
439  dtype=bool,
440  doc="Mask NAN pixels?",
441  default=True,
442  )
443  doWidenSaturationTrails = pexConfig.Field(
444  dtype=bool,
445  doc="Widen bleed trails based on their width?",
446  default=True
447  )
448 
449  # Brighter-Fatter correction.
450  doBrighterFatter = pexConfig.Field(
451  dtype=bool,
452  default=False,
453  doc="Apply the brighter fatter correction"
454  )
455  brighterFatterLevel = pexConfig.ChoiceField(
456  dtype=str,
457  default="DETECTOR",
458  doc="The level at which to correct for brighter-fatter.",
459  allowed={
460  "AMP": "Every amplifier treated separately.",
461  "DETECTOR": "One kernel per detector",
462  }
463  )
464  brighterFatterKernelFile = pexConfig.Field(
465  dtype=str,
466  default='',
467  doc="Kernel file used for the brighter fatter correction"
468  )
469  brighterFatterMaxIter = pexConfig.Field(
470  dtype=int,
471  default=10,
472  doc="Maximum number of iterations for the brighter fatter correction"
473  )
474  brighterFatterThreshold = pexConfig.Field(
475  dtype=float,
476  default=1000,
477  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
478  " absolute value of the difference between the current corrected image and the one"
479  " from the previous iteration summed over all the pixels."
480  )
481  brighterFatterApplyGain = pexConfig.Field(
482  dtype=bool,
483  default=True,
484  doc="Should the gain be applied when applying the brighter fatter correction?"
485  )
486 
487  # Dark subtraction.
488  doDark = pexConfig.Field(
489  dtype=bool,
490  doc="Apply dark frame correction?",
491  default=True,
492  )
493  darkDataProductName = pexConfig.Field(
494  dtype=str,
495  doc="Name of the dark data product",
496  default="dark",
497  )
498 
499  # Camera-specific stray light removal.
500  doStrayLight = pexConfig.Field(
501  dtype=bool,
502  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
503  default=False,
504  )
505  strayLight = pexConfig.ConfigurableField(
506  target=StrayLightTask,
507  doc="y-band stray light correction"
508  )
509 
510  # Flat correction.
511  doFlat = pexConfig.Field(
512  dtype=bool,
513  doc="Apply flat field correction?",
514  default=True,
515  )
516  flatDataProductName = pexConfig.Field(
517  dtype=str,
518  doc="Name of the flat data product",
519  default="flat",
520  )
521  flatScalingType = pexConfig.ChoiceField(
522  dtype=str,
523  doc="The method for scaling the flat on the fly.",
524  default='USER',
525  allowed={
526  "USER": "Scale by flatUserScale",
527  "MEAN": "Scale by the inverse of the mean",
528  "MEDIAN": "Scale by the inverse of the median",
529  },
530  )
531  flatUserScale = pexConfig.Field(
532  dtype=float,
533  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
534  default=1.0,
535  )
536  doTweakFlat = pexConfig.Field(
537  dtype=bool,
538  doc="Tweak flats to match observed amplifier ratios?",
539  default=False
540  )
541 
542  # Amplifier normalization based on gains instead of using flats configuration.
543  doApplyGains = pexConfig.Field(
544  dtype=bool,
545  doc="Correct the amplifiers for their gains instead of applying flat correction",
546  default=False,
547  )
548  normalizeGains = pexConfig.Field(
549  dtype=bool,
550  doc="Normalize all the amplifiers in each CCD to have the same median value.",
551  default=False,
552  )
553 
554  # Fringe correction.
555  doFringe = pexConfig.Field(
556  dtype=bool,
557  doc="Apply fringe correction?",
558  default=True,
559  )
560  fringe = pexConfig.ConfigurableField(
561  target=FringeTask,
562  doc="Fringe subtraction task",
563  )
564  fringeAfterFlat = pexConfig.Field(
565  dtype=bool,
566  doc="Do fringe subtraction after flat-fielding?",
567  default=True,
568  )
569 
570  # Distortion model application.
571  doAddDistortionModel = pexConfig.Field(
572  dtype=bool,
573  doc="Apply a distortion model based on camera geometry to the WCS?",
574  default=True,
575  )
576 
577  # Initial CCD-level background statistics options.
578  doMeasureBackground = pexConfig.Field(
579  dtype=bool,
580  doc="Measure the background level on the reduced image?",
581  default=False,
582  )
583 
584  # Camera-specific masking configuration.
585  doCameraSpecificMasking = pexConfig.Field(
586  dtype=bool,
587  doc="Mask camera-specific bad regions?",
588  default=False,
589  )
590  masking = pexConfig.ConfigurableField(
591  target=MaskingTask,
592  doc="Masking task."
593  )
594 
595  # Interpolation options.
596 
597  doInterpolate = pexConfig.Field(
598  dtype=bool,
599  doc="Interpolate masked pixels?",
600  default=True,
601  )
602  doSaturationInterpolation = pexConfig.Field(
603  dtype=bool,
604  doc="Perform interpolation over pixels masked as saturated?"
605  " NB: This is independent of doSaturation; if that is False this plane"
606  " will likely be blank, resulting in a no-op here.",
607  default=True,
608  )
609  doNanInterpolation = pexConfig.Field(
610  dtype=bool,
611  doc="Perform interpolation over pixels masked as NaN?"
612  " NB: This is independent of doNanMasking; if that is False this plane"
613  " will likely be blank, resulting in a no-op here.",
614  default=True,
615  )
616  doNanInterpAfterFlat = pexConfig.Field(
617  dtype=bool,
618  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
619  "also have to interpolate them before flat-fielding."),
620  default=False,
621  )
622  maskListToInterpolate = pexConfig.ListField(
623  dtype=str,
624  doc="List of mask planes that should be interpolated.",
625  default=['SAT', 'BAD', 'UNMASKEDNAN'],
626  )
627  doSaveInterpPixels = pexConfig.Field(
628  dtype=bool,
629  doc="Save a copy of the pre-interpolated pixel values?",
630  default=False,
631  )
632 
633  # Default photometric calibration options.
634  fluxMag0T1 = pexConfig.DictField(
635  keytype=str,
636  itemtype=float,
637  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
638  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
639  ))
640  )
641  defaultFluxMag0T1 = pexConfig.Field(
642  dtype=float,
643  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
644  default=pow(10.0, 0.4*28.0)
645  )
646 
647  # Vignette correction configuration.
648  doVignette = pexConfig.Field(
649  dtype=bool,
650  doc="Apply vignetting parameters?",
651  default=False,
652  )
653  vignette = pexConfig.ConfigurableField(
654  target=VignetteTask,
655  doc="Vignetting task.",
656  )
657 
658  # Transmission curve configuration.
659  doAttachTransmissionCurve = pexConfig.Field(
660  dtype=bool,
661  default=False,
662  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
663  )
664  doUseOpticsTransmission = pexConfig.Field(
665  dtype=bool,
666  default=True,
667  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
668  )
669  doUseFilterTransmission = pexConfig.Field(
670  dtype=bool,
671  default=True,
672  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
673  )
674  doUseSensorTransmission = pexConfig.Field(
675  dtype=bool,
676  default=True,
677  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
678  )
679  doUseAtmosphereTransmission = pexConfig.Field(
680  dtype=bool,
681  default=True,
682  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
683  )
684 
685  # Illumination correction.
686  doIlluminationCorrection = pexConfig.Field(
687  dtype=bool,
688  default=False,
689  doc="Perform illumination correction?"
690  )
691  illuminationCorrectionDataProductName = pexConfig.Field(
692  dtype=str,
693  doc="Name of the illumination correction data product.",
694  default="illumcor",
695  )
696  illumScale = pexConfig.Field(
697  dtype=float,
698  doc="Scale factor for the illumination correction.",
699  default=1.0,
700  )
701  illumFilters = pexConfig.ListField(
702  dtype=str,
703  default=[],
704  doc="Only perform illumination correction for these filters."
705  )
706 
707  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
708  doWrite = pexConfig.Field(
709  dtype=bool,
710  doc="Persist postISRCCD?",
711  default=True,
712  )
713 
714  def validate(self):
715  super().validate()
716  if self.doFlat and self.doApplyGains:
717  raise ValueError("You may not specify both doFlat and doApplyGains")
718  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
719  self.config.maskListToInterpolate.append("SAT")
720  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
721  self.config.maskListToInterpolate.append("UNMASKEDNAN")
722 
723 
724 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
725  """Apply common instrument signature correction algorithms to a raw frame.
726 
727  The process for correcting imaging data is very similar from
728  camera to camera. This task provides a vanilla implementation of
729  doing these corrections, including the ability to turn certain
730  corrections off if they are not needed. The inputs to the primary
731  method, `run()`, are a raw exposure to be corrected and the
732  calibration data products. The raw input is a single chip sized
733  mosaic of all amps including overscans and other non-science
734  pixels. The method `runDataRef()` identifies and defines the
735  calibration data products, and is intended for use by a
736  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
737  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
738  subclassed for different camera, although the most camera specific
739  methods have been split into subtasks that can be redirected
740  appropriately.
741 
742  The __init__ method sets up the subtasks for ISR processing, using
743  the defaults from `lsst.ip.isr`.
744 
745  Parameters
746  ----------
747  args : `list`
748  Positional arguments passed to the Task constructor. None used at this time.
749  kwargs : `dict`, optional
750  Keyword arguments passed on to the Task constructor. None used at this time.
751  """
752  ConfigClass = IsrTaskConfig
753  _DefaultName = "isr"
754 
755  def __init__(self, **kwargs):
756  super().__init__(**kwargs)
757  self.makeSubtask("assembleCcd")
758  self.makeSubtask("crosstalk")
759  self.makeSubtask("strayLight")
760  self.makeSubtask("fringe")
761  self.makeSubtask("masking")
762  self.makeSubtask("vignette")
763 
764  @classmethod
765  def getInputDatasetTypes(cls, config):
766  inputTypeDict = super().getInputDatasetTypes(config)
767 
768  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
769  # need because the configuration tells us we will not be bothering with the
770  # correction that uses that IDT.
771  if config.doBias is not True:
772  inputTypeDict.pop("bias", None)
773  if config.doLinearize is not True:
774  inputTypeDict.pop("linearizer", None)
775  if config.doCrosstalk is not True:
776  inputTypeDict.pop("crosstalkSources", None)
777  if config.doBrighterFatter is not True:
778  inputTypeDict.pop("bfKernel", None)
779  if config.doDefect is not True:
780  inputTypeDict.pop("defects", None)
781  if config.doDark is not True:
782  inputTypeDict.pop("dark", None)
783  if config.doFlat is not True:
784  inputTypeDict.pop("flat", None)
785  if config.doAttachTransmissionCurve is not True:
786  inputTypeDict.pop("opticsTransmission", None)
787  inputTypeDict.pop("filterTransmission", None)
788  inputTypeDict.pop("sensorTransmission", None)
789  inputTypeDict.pop("atmosphereTransmission", None)
790  if config.doUseOpticsTransmission is not True:
791  inputTypeDict.pop("opticsTransmission", None)
792  if config.doUseFilterTransmission is not True:
793  inputTypeDict.pop("filterTransmission", None)
794  if config.doUseSensorTransmission is not True:
795  inputTypeDict.pop("sensorTransmission", None)
796  if config.doUseAtmosphereTransmission is not True:
797  inputTypeDict.pop("atmosphereTransmission", None)
798  if config.doIlluminationCorrection is not True:
799  inputTypeDict.pop("illumMaskedImage", None)
800 
801  return inputTypeDict
802 
803  @classmethod
804  def getOutputDatasetTypes(cls, config):
805  outputTypeDict = super().getOutputDatasetTypes(config)
806 
807  if config.qa.doThumbnailOss is not True:
808  outputTypeDict.pop("outputOssThumbnail", None)
809  if config.qa.doThumbnailFlattened is not True:
810  outputTypeDict.pop("outputFlattenedThumbnail", None)
811  if config.doWrite is not True:
812  outputTypeDict.pop("outputExposure", None)
813 
814  return outputTypeDict
815 
816  @classmethod
817  def getPrerequisiteDatasetTypes(cls, config):
818  # Input calibration datasets should not constrain the QuantumGraph
819  # (it'd be confusing if not having flats just silently resulted in no
820  # data being processed). Our nomenclature for that is that these are
821  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
822  names = set(cls.getInputDatasetTypes(config))
823  names.remove("ccdExposure")
824  return names
825 
826  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
827  try:
828  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
829  except Exception as e:
830  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
831  (inputDataIds, e))
832 
833  inputData['isGen3'] = True
834 
835  if self.config.doLinearize is True:
836  if 'linearizer' not in inputData.keys():
837  detector = inputData['camera'][inputData['detectorNum']]
838  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
839  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
840 
841  if inputData['defects'] is not None:
842  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
843  # masking expects a list of defects defined by their bounding box
844  if not isinstance(inputData["defects"], Defects):
845  inputData["defects"] = Defects.fromTable(inputData["defects"])
846 
847  # Broken: DM-17169
848  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
849  # fixed for non-HSC cameras in the future.
850  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
851  # if self.config.doCrosstalk else None)
852 
853  # Broken: DM-17152
854  # Fringes are not tested to be handled correctly by Gen3 butler.
855  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
856  # assembler=self.assembleCcd
857  # if self.config.doAssembleIsrExposures else None)
858  # if self.config.doFringe and
859  # self.fringe.checkFilter(inputData['ccdExposure'])
860  # else pipeBase.Struct(fringes=None))
861 
862  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
863 
864  def makeDatasetType(self, dsConfig):
865  return super().makeDatasetType(dsConfig)
866 
867  def readIsrData(self, dataRef, rawExposure):
868  """!Retrieve necessary frames for instrument signature removal.
869 
870  Pre-fetching all required ISR data products limits the IO
871  required by the ISR. Any conflict between the calibration data
872  available and that needed for ISR is also detected prior to
873  doing processing, allowing it to fail quickly.
874 
875  Parameters
876  ----------
877  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
878  Butler reference of the detector data to be processed
879  rawExposure : `afw.image.Exposure`
880  The raw exposure that will later be corrected with the
881  retrieved calibration data; should not be modified in this
882  method.
883 
884  Returns
885  -------
886  result : `lsst.pipe.base.Struct`
887  Result struct with components (which may be `None`):
888  - ``bias``: bias calibration frame (`afw.image.Exposure`)
889  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
890  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
891  - ``dark``: dark calibration frame (`afw.image.Exposure`)
892  - ``flat``: flat calibration frame (`afw.image.Exposure`)
893  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
894  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
895  - ``fringes``: `lsst.pipe.base.Struct` with components:
896  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
897  - ``seed``: random seed derived from the ccdExposureId for random
898  number generator (`uint32`).
899  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
900  A ``TransmissionCurve`` that represents the throughput of the optics,
901  to be evaluated in focal-plane coordinates.
902  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
903  A ``TransmissionCurve`` that represents the throughput of the filter
904  itself, to be evaluated in focal-plane coordinates.
905  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
906  A ``TransmissionCurve`` that represents the throughput of the sensor
907  itself, to be evaluated in post-assembly trimmed detector coordinates.
908  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
909  A ``TransmissionCurve`` that represents the throughput of the
910  atmosphere, assumed to be spatially constant.
911  - ``strayLightData`` : `object`
912  An opaque object containing calibration information for
913  stray-light correction. If `None`, no correction will be
914  performed.
915  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
916 
917  Raises
918  ------
919  NotImplementedError :
920  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
921  """
922  ccd = rawExposure.getDetector()
923  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
924  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
925  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
926  if self.config.doBias else None)
927  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
928  linearizer = (dataRef.get("linearizer", immediate=True)
929  if self.doLinearize(ccd) else None)
930  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
931  if self.config.doCrosstalk else None)
932  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
933  if self.config.doDark else None)
934  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
935  if self.config.doFlat else None)
936 
937  brighterFatterKernel = None
938  if self.config.doBrighterFatter is True:
939 
940  # Use the new-style cp_pipe version of the kernel is it exists.
941  try:
942  brighterFatterKernel = dataRef.get("brighterFatterKernel")
943  except NoResults:
944  # Fall back to the old-style numpy-ndarray style kernel if necessary.
945  try:
946  brighterFatterKernel = dataRef.get("bfKernel")
947  except NoResults:
948  brighterFatterKernel = None
949  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
950  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
951  # this detector, or raise an error.
952  if self.config.brighterFatterLevel == 'DETECTOR':
953  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
954  else:
955  # TODO DM-15631 for implementing this
956  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
957 
958  defectList = (dataRef.get("defects")
959  if self.config.doDefect else None)
960  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
961  if self.config.doAssembleIsrExposures else None)
962  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
963  else pipeBase.Struct(fringes=None))
964 
965  if self.config.doAttachTransmissionCurve:
966  opticsTransmission = (dataRef.get("transmission_optics")
967  if self.config.doUseOpticsTransmission else None)
968  filterTransmission = (dataRef.get("transmission_filter")
969  if self.config.doUseFilterTransmission else None)
970  sensorTransmission = (dataRef.get("transmission_sensor")
971  if self.config.doUseSensorTransmission else None)
972  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
973  if self.config.doUseAtmosphereTransmission else None)
974  else:
975  opticsTransmission = None
976  filterTransmission = None
977  sensorTransmission = None
978  atmosphereTransmission = None
979 
980  if self.config.doStrayLight:
981  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
982  else:
983  strayLightData = None
984 
985  illumMaskedImage = (self.getIsrExposure(dataRef,
986  self.config.illuminationCorrectionDataProductName).getMaskedImage()
987  if (self.config.doIlluminationCorrection and
988  filterName in self.config.illumFilters)
989  else None)
990 
991  # Struct should include only kwargs to run()
992  return pipeBase.Struct(bias=biasExposure,
993  linearizer=linearizer,
994  crosstalkSources=crosstalkSources,
995  dark=darkExposure,
996  flat=flatExposure,
997  bfKernel=brighterFatterKernel,
998  defects=defectList,
999  fringes=fringeStruct,
1000  opticsTransmission=opticsTransmission,
1001  filterTransmission=filterTransmission,
1002  sensorTransmission=sensorTransmission,
1003  atmosphereTransmission=atmosphereTransmission,
1004  strayLightData=strayLightData,
1005  illumMaskedImage=illumMaskedImage
1006  )
1007 
1008  @pipeBase.timeMethod
1009  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1010  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
1011  opticsTransmission=None, filterTransmission=None,
1012  sensorTransmission=None, atmosphereTransmission=None,
1013  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1014  isGen3=False,
1015  ):
1016  """!Perform instrument signature removal on an exposure.
1017 
1018  Steps included in the ISR processing, in order performed, are:
1019  - saturation and suspect pixel masking
1020  - overscan subtraction
1021  - CCD assembly of individual amplifiers
1022  - bias subtraction
1023  - variance image construction
1024  - linearization of non-linear response
1025  - crosstalk masking
1026  - brighter-fatter correction
1027  - dark subtraction
1028  - fringe correction
1029  - stray light subtraction
1030  - flat correction
1031  - masking of known defects and camera specific features
1032  - vignette calculation
1033  - appending transmission curve and distortion model
1034 
1035  Parameters
1036  ----------
1037  ccdExposure : `lsst.afw.image.Exposure`
1038  The raw exposure that is to be run through ISR. The
1039  exposure is modified by this method.
1040  camera : `lsst.afw.cameraGeom.Camera`, optional
1041  The camera geometry for this exposure. Used to select the
1042  distortion model appropriate for this data.
1043  bias : `lsst.afw.image.Exposure`, optional
1044  Bias calibration frame.
1045  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1046  Functor for linearization.
1047  crosstalkSources : `list`, optional
1048  List of possible crosstalk sources.
1049  dark : `lsst.afw.image.Exposure`, optional
1050  Dark calibration frame.
1051  flat : `lsst.afw.image.Exposure`, optional
1052  Flat calibration frame.
1053  bfKernel : `numpy.ndarray`, optional
1054  Brighter-fatter kernel.
1055  defects : `lsst.meas.algorithms.Defects`, optional
1056  List of defects.
1057  fringes : `lsst.pipe.base.Struct`, optional
1058  Struct containing the fringe correction data, with
1059  elements:
1060  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1061  - ``seed``: random seed derived from the ccdExposureId for random
1062  number generator (`uint32`)
1063  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1064  A ``TransmissionCurve`` that represents the throughput of the optics,
1065  to be evaluated in focal-plane coordinates.
1066  filterTransmission : `lsst.afw.image.TransmissionCurve`
1067  A ``TransmissionCurve`` that represents the throughput of the filter
1068  itself, to be evaluated in focal-plane coordinates.
1069  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1070  A ``TransmissionCurve`` that represents the throughput of the sensor
1071  itself, to be evaluated in post-assembly trimmed detector coordinates.
1072  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1073  A ``TransmissionCurve`` that represents the throughput of the
1074  atmosphere, assumed to be spatially constant.
1075  detectorNum : `int`, optional
1076  The integer number for the detector to process.
1077  isGen3 : bool, optional
1078  Flag this call to run() as using the Gen3 butler environment.
1079  strayLightData : `object`, optional
1080  Opaque object containing calibration information for stray-light
1081  correction. If `None`, no correction will be performed.
1082  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1083  Illumination correction image.
1084 
1085  Returns
1086  -------
1087  result : `lsst.pipe.base.Struct`
1088  Result struct with component:
1089  - ``exposure`` : `afw.image.Exposure`
1090  The fully ISR corrected exposure.
1091  - ``outputExposure`` : `afw.image.Exposure`
1092  An alias for `exposure`
1093  - ``ossThumb`` : `numpy.ndarray`
1094  Thumbnail image of the exposure after overscan subtraction.
1095  - ``flattenedThumb`` : `numpy.ndarray`
1096  Thumbnail image of the exposure after flat-field correction.
1097 
1098  Raises
1099  ------
1100  RuntimeError
1101  Raised if a configuration option is set to True, but the
1102  required calibration data has not been specified.
1103 
1104  Notes
1105  -----
1106  The current processed exposure can be viewed by setting the
1107  appropriate lsstDebug entries in the `debug.display`
1108  dictionary. The names of these entries correspond to some of
1109  the IsrTaskConfig Boolean options, with the value denoting the
1110  frame to use. The exposure is shown inside the matching
1111  option check and after the processing of that step has
1112  finished. The steps with debug points are:
1113 
1114  doAssembleCcd
1115  doBias
1116  doCrosstalk
1117  doBrighterFatter
1118  doDark
1119  doFringe
1120  doStrayLight
1121  doFlat
1122 
1123  In addition, setting the "postISRCCD" entry displays the
1124  exposure after all ISR processing has finished.
1125 
1126  """
1127 
1128  if isGen3 is True:
1129  # Gen3 currently cannot automatically do configuration overrides.
1130  # DM-15257 looks to discuss this issue.
1131 
1132  self.config.doFringe = False
1133 
1134  # Configure input exposures;
1135  if detectorNum is None:
1136  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1137 
1138  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1139  bias = self.ensureExposure(bias, camera, detectorNum)
1140  dark = self.ensureExposure(dark, camera, detectorNum)
1141  flat = self.ensureExposure(flat, camera, detectorNum)
1142  else:
1143  if isinstance(ccdExposure, ButlerDataRef):
1144  return self.runDataRef(ccdExposure)
1145 
1146  ccd = ccdExposure.getDetector()
1147  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1148 
1149  if not ccd:
1150  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1151  ccd = [FakeAmp(ccdExposure, self.config)]
1152 
1153  # Validate Input
1154  if self.config.doBias and bias is None:
1155  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1156  if self.doLinearize(ccd) and linearizer is None:
1157  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1158  if self.config.doBrighterFatter and bfKernel is None:
1159  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1160  if self.config.doDark and dark is None:
1161  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1162  if self.config.doFlat and flat is None:
1163  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1164  if self.config.doDefect and defects is None:
1165  raise RuntimeError("Must supply defects if config.doDefect=True.")
1166  if self.config.doAddDistortionModel and camera is None:
1167  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1168  if (self.config.doFringe and filterName in self.fringe.config.filters and
1169  fringes.fringes is None):
1170  # The `fringes` object needs to be a pipeBase.Struct, as
1171  # we use it as a `dict` for the parameters of
1172  # `FringeTask.run()`. The `fringes.fringes` `list` may
1173  # not be `None` if `doFringe=True`. Otherwise, raise.
1174  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1175  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1176  illumMaskedImage is None):
1177  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1178 
1179  # Begin ISR processing.
1180  if self.config.doConvertIntToFloat:
1181  self.log.info("Converting exposure to floating point values.")
1182  ccdExposure = self.convertIntToFloat(ccdExposure)
1183 
1184  # Amplifier level processing.
1185  overscans = []
1186  for amp in ccd:
1187  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1188  if ccdExposure.getBBox().contains(amp.getBBox()):
1189  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1190  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1191 
1192  if self.config.doOverscan and not badAmp:
1193  # Overscan correction on amp-by-amp basis.
1194  overscanResults = self.overscanCorrection(ccdExposure, amp)
1195  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1196  if overscanResults is not None and \
1197  self.config.qa is not None and self.config.qa.saveStats is True:
1198  if isinstance(overscanResults.overscanFit, float):
1199  qaMedian = overscanResults.overscanFit
1200  qaStdev = float("NaN")
1201  else:
1202  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1203  afwMath.MEDIAN | afwMath.STDEVCLIP)
1204  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1205  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1206 
1207  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1208  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1209  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1210  amp.getName(), qaMedian, qaStdev)
1211  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1212  else:
1213  if badAmp:
1214  self.log.warn("Amplifier %s is bad.", amp.getName())
1215  overscanResults = None
1216 
1217  overscans.append(overscanResults if overscanResults is not None else None)
1218  else:
1219  self.log.info("Skipped OSCAN for %s.", amp.getName())
1220 
1221  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1222  self.log.info("Applying crosstalk correction.")
1223  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1224  self.debugView(ccdExposure, "doCrosstalk")
1225 
1226  if self.config.doAssembleCcd:
1227  self.log.info("Assembling CCD from amplifiers.")
1228  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1229 
1230  if self.config.expectWcs and not ccdExposure.getWcs():
1231  self.log.warn("No WCS found in input exposure.")
1232  self.debugView(ccdExposure, "doAssembleCcd")
1233 
1234  ossThumb = None
1235  if self.config.qa.doThumbnailOss:
1236  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1237 
1238  if self.config.doBias:
1239  self.log.info("Applying bias correction.")
1240  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1241  trimToFit=self.config.doTrimToMatchCalib)
1242  self.debugView(ccdExposure, "doBias")
1243 
1244  if self.config.doVariance:
1245  for amp, overscanResults in zip(ccd, overscans):
1246  if ccdExposure.getBBox().contains(amp.getBBox()):
1247  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1248  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1249  if overscanResults is not None:
1250  self.updateVariance(ampExposure, amp,
1251  overscanImage=overscanResults.overscanImage)
1252  else:
1253  self.updateVariance(ampExposure, amp,
1254  overscanImage=None)
1255  if self.config.qa is not None and self.config.qa.saveStats is True:
1256  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1257  afwMath.MEDIAN | afwMath.STDEVCLIP)
1258  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1259  qaStats.getValue(afwMath.MEDIAN))
1260  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1261  qaStats.getValue(afwMath.STDEVCLIP))
1262  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1263  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1264  qaStats.getValue(afwMath.STDEVCLIP))
1265 
1266  if self.doLinearize(ccd):
1267  self.log.info("Applying linearizer.")
1268  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1269 
1270  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1271  self.log.info("Applying crosstalk correction.")
1272  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1273  self.debugView(ccdExposure, "doCrosstalk")
1274 
1275  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1276  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1277  if self.config.doDefect:
1278  self.log.info("Masking defects.")
1279  self.maskDefect(ccdExposure, defects)
1280 
1281  if self.config.numEdgeSuspect > 0:
1282  self.log.info("Masking edges as SUSPECT.")
1283  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1284  maskPlane="SUSPECT")
1285 
1286  if self.config.doNanMasking:
1287  self.log.info("Masking NAN value pixels.")
1288  self.maskNan(ccdExposure)
1289 
1290  if self.config.doWidenSaturationTrails:
1291  self.log.info("Widening saturation trails.")
1292  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1293 
1294  if self.config.doCameraSpecificMasking:
1295  self.log.info("Masking regions for camera specific reasons.")
1296  self.masking.run(ccdExposure)
1297 
1298  if self.config.doBrighterFatter:
1299  # We need to apply flats and darks before we can interpolate, and we
1300  # need to interpolate before we do B-F, but we do B-F without the
1301  # flats and darks applied so we can work in units of electrons or holes.
1302  # This context manager applies and then removes the darks and flats.
1303  #
1304  # We also do not want to interpolate values here, so operate on temporary
1305  # images so we can apply only the BF-correction and roll back the
1306  # interpolation.
1307  interpExp = ccdExposure.clone()
1308  with self.flatContext(interpExp, flat, dark):
1309  isrFunctions.interpolateFromMask(
1310  maskedImage=interpExp.getMaskedImage(),
1311  fwhm=self.config.fwhm,
1312  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1313  maskNameList=self.config.maskListToInterpolate
1314  )
1315  bfExp = interpExp.clone()
1316 
1317  self.log.info("Applying brighter fatter correction.")
1318  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1319  self.config.brighterFatterMaxIter,
1320  self.config.brighterFatterThreshold,
1321  self.config.brighterFatterApplyGain
1322  )
1323  if bfResults[1] == self.config.brighterFatterMaxIter:
1324  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1325  bfResults[0])
1326  else:
1327  self.log.info("Finished brighter fatter correction in %d iterations.",
1328  bfResults[1])
1329  image = ccdExposure.getMaskedImage().getImage()
1330  bfCorr = bfExp.getMaskedImage().getImage()
1331  bfCorr -= interpExp.getMaskedImage().getImage()
1332  image += bfCorr
1333 
1334  # Applying the brighter-fatter correction applies a
1335  # convolution to the science image. At the edges this
1336  # convolution may not have sufficient valid pixels to
1337  # produce a valid correction. Mark pixels within the size
1338  # of the brighter-fatter kernel as EDGE to warn of this
1339  # fact.
1340  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1341  maskPlane="EDGE")
1342  self.log.warn("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1343 
1344  self.debugView(ccdExposure, "doBrighterFatter")
1345 
1346  if self.config.doDark:
1347  self.log.info("Applying dark correction.")
1348  self.darkCorrection(ccdExposure, dark)
1349  self.debugView(ccdExposure, "doDark")
1350 
1351  if self.config.doFringe and not self.config.fringeAfterFlat:
1352  self.log.info("Applying fringe correction before flat.")
1353  self.fringe.run(ccdExposure, **fringes.getDict())
1354  self.debugView(ccdExposure, "doFringe")
1355 
1356  if self.config.doStrayLight:
1357  if strayLightData is not None:
1358  self.log.info("Applying stray light correction.")
1359  self.strayLight.run(ccdExposure, strayLightData)
1360  self.debugView(ccdExposure, "doStrayLight")
1361  else:
1362  self.log.debug("Skipping stray light correction: no data found for this image.")
1363 
1364  if self.config.doFlat:
1365  self.log.info("Applying flat correction.")
1366  self.flatCorrection(ccdExposure, flat)
1367  self.debugView(ccdExposure, "doFlat")
1368 
1369  if self.config.doApplyGains:
1370  self.log.info("Applying gain correction instead of flat.")
1371  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1372 
1373  if self.config.doFringe and self.config.fringeAfterFlat:
1374  self.log.info("Applying fringe correction after flat.")
1375  self.fringe.run(ccdExposure, **fringes.getDict())
1376 
1377  if self.config.doVignette:
1378  self.log.info("Constructing Vignette polygon.")
1379  self.vignettePolygon = self.vignette.run(ccdExposure)
1380 
1381  if self.config.vignette.doWriteVignettePolygon:
1382  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1383 
1384  if self.config.doAttachTransmissionCurve:
1385  self.log.info("Adding transmission curves.")
1386  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1387  filterTransmission=filterTransmission,
1388  sensorTransmission=sensorTransmission,
1389  atmosphereTransmission=atmosphereTransmission)
1390 
1391  if self.config.doAddDistortionModel:
1392  self.log.info("Adding a distortion model to the WCS.")
1393  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1394 
1395  flattenedThumb = None
1396  if self.config.qa.doThumbnailFlattened:
1397  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1398 
1399  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1400  self.log.info("Performing illumination correction.")
1401  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1402  illumMaskedImage, illumScale=self.config.illumScale,
1403  trimToFit=self.config.doTrimToMatchCalib)
1404 
1405  preInterpExp = None
1406  if self.config.doSaveInterpPixels:
1407  preInterpExp = ccdExposure.clone()
1408 
1409  # Reset and interpolate bad pixels.
1410  #
1411  # Large contiguous bad regions (which should have the BAD mask
1412  # bit set) should have their values set to the image median.
1413  # This group should include defects and bad amplifiers. As the
1414  # area covered by these defects are large, there's little
1415  # reason to expect that interpolation would provide a more
1416  # useful value.
1417  #
1418  # Smaller defects can be safely interpolated after the larger
1419  # regions have had their pixel values reset. This ensures
1420  # that the remaining defects adjacent to bad amplifiers (as an
1421  # example) do not attempt to interpolate extreme values.
1422  if self.config.doSetBadRegions:
1423  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1424  if badPixelCount > 0:
1425  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1426 
1427  if self.config.doInterpolate:
1428  self.log.info("Interpolating masked pixels.")
1429  isrFunctions.interpolateFromMask(
1430  maskedImage=ccdExposure.getMaskedImage(),
1431  fwhm=self.config.fwhm,
1432  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1433  maskNameList=list(self.config.maskListToInterpolate)
1434  )
1435 
1436  self.roughZeroPoint(ccdExposure)
1437 
1438  if self.config.doMeasureBackground:
1439  self.log.info("Measuring background level.")
1440  self.measureBackground(ccdExposure, self.config.qa)
1441 
1442  if self.config.qa is not None and self.config.qa.saveStats is True:
1443  for amp in ccd:
1444  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1445  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1446  afwMath.MEDIAN | afwMath.STDEVCLIP)
1447  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1448  qaStats.getValue(afwMath.MEDIAN))
1449  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1450  qaStats.getValue(afwMath.STDEVCLIP))
1451  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1452  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1453  qaStats.getValue(afwMath.STDEVCLIP))
1454 
1455  self.debugView(ccdExposure, "postISRCCD")
1456 
1457  return pipeBase.Struct(
1458  exposure=ccdExposure,
1459  ossThumb=ossThumb,
1460  flattenedThumb=flattenedThumb,
1461 
1462  preInterpolatedExposure=preInterpExp,
1463  outputExposure=ccdExposure,
1464  outputOssThumbnail=ossThumb,
1465  outputFlattenedThumbnail=flattenedThumb,
1466  )
1467 
1468  @pipeBase.timeMethod
1469  def runDataRef(self, sensorRef):
1470  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1471 
1472  This method contains the `CmdLineTask` interface to the ISR
1473  processing. All IO is handled here, freeing the `run()` method
1474  to manage only pixel-level calculations. The steps performed
1475  are:
1476  - Read in necessary detrending/isr/calibration data.
1477  - Process raw exposure in `run()`.
1478  - Persist the ISR-corrected exposure as "postISRCCD" if
1479  config.doWrite=True.
1480 
1481  Parameters
1482  ----------
1483  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1484  DataRef of the detector data to be processed
1485 
1486  Returns
1487  -------
1488  result : `lsst.pipe.base.Struct`
1489  Result struct with component:
1490  - ``exposure`` : `afw.image.Exposure`
1491  The fully ISR corrected exposure.
1492 
1493  Raises
1494  ------
1495  RuntimeError
1496  Raised if a configuration option is set to True, but the
1497  required calibration data does not exist.
1498 
1499  """
1500  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1501 
1502  ccdExposure = sensorRef.get(self.config.datasetType)
1503 
1504  camera = sensorRef.get("camera")
1505  if camera is None and self.config.doAddDistortionModel:
1506  raise RuntimeError("config.doAddDistortionModel is True "
1507  "but could not get a camera from the butler.")
1508  isrData = self.readIsrData(sensorRef, ccdExposure)
1509 
1510  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1511 
1512  if self.config.doWrite:
1513  sensorRef.put(result.exposure, "postISRCCD")
1514  if result.preInterpolatedExposure is not None:
1515  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1516  if result.ossThumb is not None:
1517  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1518  if result.flattenedThumb is not None:
1519  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1520 
1521  return result
1522 
1523  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1524  """!Retrieve a calibration dataset for removing instrument signature.
1525 
1526  Parameters
1527  ----------
1528 
1529  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1530  DataRef of the detector data to find calibration datasets
1531  for.
1532  datasetType : `str`
1533  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1534  immediate : `Bool`
1535  If True, disable butler proxies to enable error handling
1536  within this routine.
1537 
1538  Returns
1539  -------
1540  exposure : `lsst.afw.image.Exposure`
1541  Requested calibration frame.
1542 
1543  Raises
1544  ------
1545  RuntimeError
1546  Raised if no matching calibration frame can be found.
1547  """
1548  try:
1549  exp = dataRef.get(datasetType, immediate=immediate)
1550  except Exception as exc1:
1551  if not self.config.fallbackFilterName:
1552  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1553  try:
1554  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1555  except Exception as exc2:
1556  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1557  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1558  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1559 
1560  if self.config.doAssembleIsrExposures:
1561  exp = self.assembleCcd.assembleCcd(exp)
1562  return exp
1563 
1564  def ensureExposure(self, inputExp, camera, detectorNum):
1565  """Ensure that the data returned by Butler is a fully constructed exposure.
1566 
1567  ISR requires exposure-level image data for historical reasons, so if we did
1568  not recieve that from Butler, construct it from what we have, modifying the
1569  input in place.
1570 
1571  Parameters
1572  ----------
1573  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1574  `lsst.afw.image.ImageF`
1575  The input data structure obtained from Butler.
1576  camera : `lsst.afw.cameraGeom.camera`
1577  The camera associated with the image. Used to find the appropriate
1578  detector.
1579  detectorNum : `int`
1580  The detector this exposure should match.
1581 
1582  Returns
1583  -------
1584  inputExp : `lsst.afw.image.Exposure`
1585  The re-constructed exposure, with appropriate detector parameters.
1586 
1587  Raises
1588  ------
1589  TypeError
1590  Raised if the input data cannot be used to construct an exposure.
1591  """
1592  if isinstance(inputExp, afwImage.DecoratedImageU):
1593  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1594  elif isinstance(inputExp, afwImage.ImageF):
1595  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1596  elif isinstance(inputExp, afwImage.MaskedImageF):
1597  inputExp = afwImage.makeExposure(inputExp)
1598  elif isinstance(inputExp, afwImage.Exposure):
1599  pass
1600  elif inputExp is None:
1601  # Assume this will be caught by the setup if it is a problem.
1602  return inputExp
1603  else:
1604  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1605  (type(inputExp), ))
1606 
1607  if inputExp.getDetector() is None:
1608  inputExp.setDetector(camera[detectorNum])
1609 
1610  return inputExp
1611 
1612  def convertIntToFloat(self, exposure):
1613  """Convert exposure image from uint16 to float.
1614 
1615  If the exposure does not need to be converted, the input is
1616  immediately returned. For exposures that are converted to use
1617  floating point pixels, the variance is set to unity and the
1618  mask to zero.
1619 
1620  Parameters
1621  ----------
1622  exposure : `lsst.afw.image.Exposure`
1623  The raw exposure to be converted.
1624 
1625  Returns
1626  -------
1627  newexposure : `lsst.afw.image.Exposure`
1628  The input ``exposure``, converted to floating point pixels.
1629 
1630  Raises
1631  ------
1632  RuntimeError
1633  Raised if the exposure type cannot be converted to float.
1634 
1635  """
1636  if isinstance(exposure, afwImage.ExposureF):
1637  # Nothing to be done
1638  self.log.debug("Exposure already of type float.")
1639  return exposure
1640  if not hasattr(exposure, "convertF"):
1641  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1642 
1643  newexposure = exposure.convertF()
1644  newexposure.variance[:] = 1
1645  newexposure.mask[:] = 0x0
1646 
1647  return newexposure
1648 
1649  def maskAmplifier(self, ccdExposure, amp, defects):
1650  """Identify bad amplifiers, saturated and suspect pixels.
1651 
1652  Parameters
1653  ----------
1654  ccdExposure : `lsst.afw.image.Exposure`
1655  Input exposure to be masked.
1656  amp : `lsst.afw.table.AmpInfoCatalog`
1657  Catalog of parameters defining the amplifier on this
1658  exposure to mask.
1659  defects : `lsst.meas.algorithms.Defects`
1660  List of defects. Used to determine if the entire
1661  amplifier is bad.
1662 
1663  Returns
1664  -------
1665  badAmp : `Bool`
1666  If this is true, the entire amplifier area is covered by
1667  defects and unusable.
1668 
1669  """
1670  maskedImage = ccdExposure.getMaskedImage()
1671 
1672  badAmp = False
1673 
1674  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1675  # comparison with current defects definition.
1676  if defects is not None:
1677  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1678 
1679  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1680  # association with pixels in current ccdExposure).
1681  if badAmp:
1682  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1683  afwImage.PARENT)
1684  maskView = dataView.getMask()
1685  maskView |= maskView.getPlaneBitMask("BAD")
1686  del maskView
1687  return badAmp
1688 
1689  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1690  # Saturation and suspect pixels can be masked now, though.
1691  limits = dict()
1692  if self.config.doSaturation and not badAmp:
1693  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1694  if self.config.doSuspect and not badAmp:
1695  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1696  if math.isfinite(self.config.saturation):
1697  limits.update({self.config.saturatedMaskName: self.config.saturation})
1698 
1699  for maskName, maskThreshold in limits.items():
1700  if not math.isnan(maskThreshold):
1701  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1702  isrFunctions.makeThresholdMask(
1703  maskedImage=dataView,
1704  threshold=maskThreshold,
1705  growFootprints=0,
1706  maskName=maskName
1707  )
1708 
1709  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1710  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1711  afwImage.PARENT)
1712  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1713  self.config.suspectMaskName])
1714  if numpy.all(maskView.getArray() & maskVal > 0):
1715  badAmp = True
1716  maskView |= maskView.getPlaneBitMask("BAD")
1717 
1718  return badAmp
1719 
1720  def overscanCorrection(self, ccdExposure, amp):
1721  """Apply overscan correction in place.
1722 
1723  This method does initial pixel rejection of the overscan
1724  region. The overscan can also be optionally segmented to
1725  allow for discontinuous overscan responses to be fit
1726  separately. The actual overscan subtraction is performed by
1727  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1728  which is called here after the amplifier is preprocessed.
1729 
1730  Parameters
1731  ----------
1732  ccdExposure : `lsst.afw.image.Exposure`
1733  Exposure to have overscan correction performed.
1734  amp : `lsst.afw.table.AmpInfoCatalog`
1735  The amplifier to consider while correcting the overscan.
1736 
1737  Returns
1738  -------
1739  overscanResults : `lsst.pipe.base.Struct`
1740  Result struct with components:
1741  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1742  Value or fit subtracted from the amplifier image data.
1743  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1744  Value or fit subtracted from the overscan image data.
1745  - ``overscanImage`` : `lsst.afw.image.Image`
1746  Image of the overscan region with the overscan
1747  correction applied. This quantity is used to estimate
1748  the amplifier read noise empirically.
1749 
1750  Raises
1751  ------
1752  RuntimeError
1753  Raised if the ``amp`` does not contain raw pixel information.
1754 
1755  See Also
1756  --------
1757  lsst.ip.isr.isrFunctions.overscanCorrection
1758  """
1759  if not amp.getHasRawInfo():
1760  raise RuntimeError("This method must be executed on an amp with raw information.")
1761 
1762  if amp.getRawHorizontalOverscanBBox().isEmpty():
1763  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1764  return None
1765 
1766  statControl = afwMath.StatisticsControl()
1767  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1768 
1769  # Determine the bounding boxes
1770  dataBBox = amp.getRawDataBBox()
1771  oscanBBox = amp.getRawHorizontalOverscanBBox()
1772  dx0 = 0
1773  dx1 = 0
1774 
1775  prescanBBox = amp.getRawPrescanBBox()
1776  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1777  dx0 += self.config.overscanNumLeadingColumnsToSkip
1778  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1779  else:
1780  dx0 += self.config.overscanNumTrailingColumnsToSkip
1781  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1782 
1783  # Determine if we need to work on subregions of the amplifier and overscan.
1784  imageBBoxes = []
1785  overscanBBoxes = []
1786 
1787  if ((self.config.overscanBiasJump and
1788  self.config.overscanBiasJumpLocation) and
1789  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1790  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1791  self.config.overscanBiasJumpDevices)):
1792  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1793  yLower = self.config.overscanBiasJumpLocation
1794  yUpper = dataBBox.getHeight() - yLower
1795  else:
1796  yUpper = self.config.overscanBiasJumpLocation
1797  yLower = dataBBox.getHeight() - yUpper
1798 
1799  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1800  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1801  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1802  lsst.geom.Extent2I(dx0, 0),
1803  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1804  yLower)))
1805 
1806  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1807  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1808  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1809  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1810  yUpper)))
1811  else:
1812  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1813  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1814  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1815  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1816  oscanBBox.getHeight())))
1817 
1818  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1819  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1820  ampImage = ccdExposure.maskedImage[imageBBox]
1821  overscanImage = ccdExposure.maskedImage[overscanBBox]
1822 
1823  overscanArray = overscanImage.image.array
1824  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1825  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1826  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1827 
1828  statControl = afwMath.StatisticsControl()
1829  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1830 
1831  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1832  overscanImage=overscanImage,
1833  fitType=self.config.overscanFitType,
1834  order=self.config.overscanOrder,
1835  collapseRej=self.config.overscanNumSigmaClip,
1836  statControl=statControl,
1837  overscanIsInt=self.config.overscanIsInt
1838  )
1839 
1840  # Measure average overscan levels and record them in the metadata.
1841  levelStat = afwMath.MEDIAN
1842  sigmaStat = afwMath.STDEVCLIP
1843 
1844  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1845  self.config.qa.flatness.nIter)
1846  metadata = ccdExposure.getMetadata()
1847  ampNum = amp.getName()
1848  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1849  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1850  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1851  else:
1852  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1853  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1854  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1855 
1856  return overscanResults
1857 
1858  def updateVariance(self, ampExposure, amp, overscanImage=None):
1859  """Set the variance plane using the amplifier gain and read noise
1860 
1861  The read noise is calculated from the ``overscanImage`` if the
1862  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1863  the value from the amplifier data is used.
1864 
1865  Parameters
1866  ----------
1867  ampExposure : `lsst.afw.image.Exposure`
1868  Exposure to process.
1869  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1870  Amplifier detector data.
1871  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1872  Image of overscan, required only for empirical read noise.
1873 
1874  See also
1875  --------
1876  lsst.ip.isr.isrFunctions.updateVariance
1877  """
1878  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1879  gain = amp.getGain()
1880 
1881  if math.isnan(gain):
1882  gain = 1.0
1883  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1884  elif gain <= 0:
1885  patchedGain = 1.0
1886  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1887  amp.getName(), gain, patchedGain)
1888  gain = patchedGain
1889 
1890  if self.config.doEmpiricalReadNoise and overscanImage is None:
1891  self.log.info("Overscan is none for EmpiricalReadNoise.")
1892 
1893  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1894  stats = afwMath.StatisticsControl()
1895  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1896  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1897  self.log.info("Calculated empirical read noise for amp %s: %f.",
1898  amp.getName(), readNoise)
1899  else:
1900  readNoise = amp.getReadNoise()
1901 
1902  isrFunctions.updateVariance(
1903  maskedImage=ampExposure.getMaskedImage(),
1904  gain=gain,
1905  readNoise=readNoise,
1906  )
1907 
1908  def darkCorrection(self, exposure, darkExposure, invert=False):
1909  """!Apply dark correction in place.
1910 
1911  Parameters
1912  ----------
1913  exposure : `lsst.afw.image.Exposure`
1914  Exposure to process.
1915  darkExposure : `lsst.afw.image.Exposure`
1916  Dark exposure of the same size as ``exposure``.
1917  invert : `Bool`, optional
1918  If True, re-add the dark to an already corrected image.
1919 
1920  Raises
1921  ------
1922  RuntimeError
1923  Raised if either ``exposure`` or ``darkExposure`` do not
1924  have their dark time defined.
1925 
1926  See Also
1927  --------
1928  lsst.ip.isr.isrFunctions.darkCorrection
1929  """
1930  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1931  if math.isnan(expScale):
1932  raise RuntimeError("Exposure darktime is NAN.")
1933  if darkExposure.getInfo().getVisitInfo() is not None:
1934  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1935  else:
1936  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1937  # so getDarkTime() does not exist.
1938  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1939  darkScale = 1.0
1940 
1941  if math.isnan(darkScale):
1942  raise RuntimeError("Dark calib darktime is NAN.")
1943  isrFunctions.darkCorrection(
1944  maskedImage=exposure.getMaskedImage(),
1945  darkMaskedImage=darkExposure.getMaskedImage(),
1946  expScale=expScale,
1947  darkScale=darkScale,
1948  invert=invert,
1949  trimToFit=self.config.doTrimToMatchCalib
1950  )
1951 
1952  def doLinearize(self, detector):
1953  """!Check if linearization is needed for the detector cameraGeom.
1954 
1955  Checks config.doLinearize and the linearity type of the first
1956  amplifier.
1957 
1958  Parameters
1959  ----------
1960  detector : `lsst.afw.cameraGeom.Detector`
1961  Detector to get linearity type from.
1962 
1963  Returns
1964  -------
1965  doLinearize : `Bool`
1966  If True, linearization should be performed.
1967  """
1968  return self.config.doLinearize and \
1969  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1970 
1971  def flatCorrection(self, exposure, flatExposure, invert=False):
1972  """!Apply flat correction in place.
1973 
1974  Parameters
1975  ----------
1976  exposure : `lsst.afw.image.Exposure`
1977  Exposure to process.
1978  flatExposure : `lsst.afw.image.Exposure`
1979  Flat exposure of the same size as ``exposure``.
1980  invert : `Bool`, optional
1981  If True, unflatten an already flattened image.
1982 
1983  See Also
1984  --------
1985  lsst.ip.isr.isrFunctions.flatCorrection
1986  """
1987  isrFunctions.flatCorrection(
1988  maskedImage=exposure.getMaskedImage(),
1989  flatMaskedImage=flatExposure.getMaskedImage(),
1990  scalingType=self.config.flatScalingType,
1991  userScale=self.config.flatUserScale,
1992  invert=invert,
1993  trimToFit=self.config.doTrimToMatchCalib
1994  )
1995 
1996  def saturationDetection(self, exposure, amp):
1997  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1998 
1999  Parameters
2000  ----------
2001  exposure : `lsst.afw.image.Exposure`
2002  Exposure to process. Only the amplifier DataSec is processed.
2003  amp : `lsst.afw.table.AmpInfoCatalog`
2004  Amplifier detector data.
2005 
2006  See Also
2007  --------
2008  lsst.ip.isr.isrFunctions.makeThresholdMask
2009  """
2010  if not math.isnan(amp.getSaturation()):
2011  maskedImage = exposure.getMaskedImage()
2012  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2013  isrFunctions.makeThresholdMask(
2014  maskedImage=dataView,
2015  threshold=amp.getSaturation(),
2016  growFootprints=0,
2017  maskName=self.config.saturatedMaskName,
2018  )
2019 
2020  def saturationInterpolation(self, exposure):
2021  """!Interpolate over saturated pixels, in place.
2022 
2023  This method should be called after `saturationDetection`, to
2024  ensure that the saturated pixels have been identified in the
2025  SAT mask. It should also be called after `assembleCcd`, since
2026  saturated regions may cross amplifier boundaries.
2027 
2028  Parameters
2029  ----------
2030  exposure : `lsst.afw.image.Exposure`
2031  Exposure to process.
2032 
2033  See Also
2034  --------
2035  lsst.ip.isr.isrTask.saturationDetection
2036  lsst.ip.isr.isrFunctions.interpolateFromMask
2037  """
2038  isrFunctions.interpolateFromMask(
2039  maskedImage=exposure.getMaskedImage(),
2040  fwhm=self.config.fwhm,
2041  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2042  maskNameList=list(self.config.saturatedMaskName),
2043  )
2044 
2045  def suspectDetection(self, exposure, amp):
2046  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2047 
2048  Parameters
2049  ----------
2050  exposure : `lsst.afw.image.Exposure`
2051  Exposure to process. Only the amplifier DataSec is processed.
2052  amp : `lsst.afw.table.AmpInfoCatalog`
2053  Amplifier detector data.
2054 
2055  See Also
2056  --------
2057  lsst.ip.isr.isrFunctions.makeThresholdMask
2058 
2059  Notes
2060  -----
2061  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2062  This is intended to indicate pixels that may be affected by unknown systematics;
2063  for example if non-linearity corrections above a certain level are unstable
2064  then that would be a useful value for suspectLevel. A value of `nan` indicates
2065  that no such level exists and no pixels are to be masked as suspicious.
2066  """
2067  suspectLevel = amp.getSuspectLevel()
2068  if math.isnan(suspectLevel):
2069  return
2070 
2071  maskedImage = exposure.getMaskedImage()
2072  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2073  isrFunctions.makeThresholdMask(
2074  maskedImage=dataView,
2075  threshold=suspectLevel,
2076  growFootprints=0,
2077  maskName=self.config.suspectMaskName,
2078  )
2079 
2080  def maskDefect(self, exposure, defectBaseList):
2081  """!Mask defects using mask plane "BAD", in place.
2082 
2083  Parameters
2084  ----------
2085  exposure : `lsst.afw.image.Exposure`
2086  Exposure to process.
2087  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2088  `lsst.afw.image.DefectBase`.
2089  List of defects to mask.
2090 
2091  Notes
2092  -----
2093  Call this after CCD assembly, since defects may cross amplifier boundaries.
2094  """
2095  maskedImage = exposure.getMaskedImage()
2096  if not isinstance(defectBaseList, Defects):
2097  # Promotes DefectBase to Defect
2098  defectList = Defects(defectBaseList)
2099  else:
2100  defectList = defectBaseList
2101  defectList.maskPixels(maskedImage, maskName="BAD")
2102 
2103  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2104  """!Mask edge pixels with applicable mask plane.
2105 
2106  Parameters
2107  ----------
2108  exposure : `lsst.afw.image.Exposure`
2109  Exposure to process.
2110  numEdgePixels : `int`, optional
2111  Number of edge pixels to mask.
2112  maskPlane : `str`, optional
2113  Mask plane name to use.
2114  """
2115  maskedImage = exposure.getMaskedImage()
2116  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2117 
2118  if numEdgePixels > 0:
2119  goodBBox = maskedImage.getBBox()
2120  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2121  goodBBox.grow(-numEdgePixels)
2122  # Mask pixels outside goodBBox
2123  SourceDetectionTask.setEdgeBits(
2124  maskedImage,
2125  goodBBox,
2126  maskBitMask
2127  )
2128 
2129  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2130  """Mask and interpolate defects using mask plane "BAD", in place.
2131 
2132  Parameters
2133  ----------
2134  exposure : `lsst.afw.image.Exposure`
2135  Exposure to process.
2136  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2137  `lsst.afw.image.DefectBase`.
2138  List of defects to mask and interpolate.
2139 
2140  See Also
2141  --------
2142  lsst.ip.isr.isrTask.maskDefect()
2143  """
2144  self.maskDefect(exposure, defectBaseList)
2145  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2146  maskPlane="SUSPECT")
2147  isrFunctions.interpolateFromMask(
2148  maskedImage=exposure.getMaskedImage(),
2149  fwhm=self.config.fwhm,
2150  growSaturatedFootprints=0,
2151  maskNameList=["BAD"],
2152  )
2153 
2154  def maskNan(self, exposure):
2155  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2156 
2157  Parameters
2158  ----------
2159  exposure : `lsst.afw.image.Exposure`
2160  Exposure to process.
2161 
2162  Notes
2163  -----
2164  We mask over all NaNs, including those that are masked with
2165  other bits (because those may or may not be interpolated over
2166  later, and we want to remove all NaNs). Despite this
2167  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2168  the historical name.
2169  """
2170  maskedImage = exposure.getMaskedImage()
2171 
2172  # Find and mask NaNs
2173  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2174  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2175  numNans = maskNans(maskedImage, maskVal)
2176  self.metadata.set("NUMNANS", numNans)
2177  if numNans > 0:
2178  self.log.warn("There were %d unmasked NaNs.", numNans)
2179 
2180  def maskAndInterpolateNan(self, exposure):
2181  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2182 
2183  Parameters
2184  ----------
2185  exposure : `lsst.afw.image.Exposure`
2186  Exposure to process.
2187 
2188  See Also
2189  --------
2190  lsst.ip.isr.isrTask.maskNan()
2191  """
2192  self.maskNan(exposure)
2193  isrFunctions.interpolateFromMask(
2194  maskedImage=exposure.getMaskedImage(),
2195  fwhm=self.config.fwhm,
2196  growSaturatedFootprints=0,
2197  maskNameList=["UNMASKEDNAN"],
2198  )
2199 
2200  def measureBackground(self, exposure, IsrQaConfig=None):
2201  """Measure the image background in subgrids, for quality control purposes.
2202 
2203  Parameters
2204  ----------
2205  exposure : `lsst.afw.image.Exposure`
2206  Exposure to process.
2207  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2208  Configuration object containing parameters on which background
2209  statistics and subgrids to use.
2210  """
2211  if IsrQaConfig is not None:
2212  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2213  IsrQaConfig.flatness.nIter)
2214  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2215  statsControl.setAndMask(maskVal)
2216  maskedImage = exposure.getMaskedImage()
2217  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2218  skyLevel = stats.getValue(afwMath.MEDIAN)
2219  skySigma = stats.getValue(afwMath.STDEVCLIP)
2220  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2221  metadata = exposure.getMetadata()
2222  metadata.set('SKYLEVEL', skyLevel)
2223  metadata.set('SKYSIGMA', skySigma)
2224 
2225  # calcluating flatlevel over the subgrids
2226  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2227  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2228  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2229  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2230  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2231  skyLevels = numpy.zeros((nX, nY))
2232 
2233  for j in range(nY):
2234  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2235  for i in range(nX):
2236  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2237 
2238  xLLC = xc - meshXHalf
2239  yLLC = yc - meshYHalf
2240  xURC = xc + meshXHalf - 1
2241  yURC = yc + meshYHalf - 1
2242 
2243  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2244  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2245 
2246  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2247 
2248  good = numpy.where(numpy.isfinite(skyLevels))
2249  skyMedian = numpy.median(skyLevels[good])
2250  flatness = (skyLevels[good] - skyMedian) / skyMedian
2251  flatness_rms = numpy.std(flatness)
2252  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2253 
2254  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2255  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2256  nX, nY, flatness_pp, flatness_rms)
2257 
2258  metadata.set('FLATNESS_PP', float(flatness_pp))
2259  metadata.set('FLATNESS_RMS', float(flatness_rms))
2260  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2261  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2262  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2263 
2264  def roughZeroPoint(self, exposure):
2265  """Set an approximate magnitude zero point for the exposure.
2266 
2267  Parameters
2268  ----------
2269  exposure : `lsst.afw.image.Exposure`
2270  Exposure to process.
2271  """
2272  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2273  if filterName in self.config.fluxMag0T1:
2274  fluxMag0 = self.config.fluxMag0T1[filterName]
2275  else:
2276  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2277  fluxMag0 = self.config.defaultFluxMag0T1
2278 
2279  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2280  if not expTime > 0: # handle NaN as well as <= 0
2281  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2282  return
2283 
2284  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2285  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2286 
2287  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2288  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2289 
2290  Parameters
2291  ----------
2292  ccdExposure : `lsst.afw.image.Exposure`
2293  Exposure to process.
2294  fpPolygon : `lsst.afw.geom.Polygon`
2295  Polygon in focal plane coordinates.
2296  """
2297  # Get ccd corners in focal plane coordinates
2298  ccd = ccdExposure.getDetector()
2299  fpCorners = ccd.getCorners(FOCAL_PLANE)
2300  ccdPolygon = Polygon(fpCorners)
2301 
2302  # Get intersection of ccd corners with fpPolygon
2303  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2304 
2305  # Transform back to pixel positions and build new polygon
2306  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2307  validPolygon = Polygon(ccdPoints)
2308  ccdExposure.getInfo().setValidPolygon(validPolygon)
2309 
2310  @contextmanager
2311  def flatContext(self, exp, flat, dark=None):
2312  """Context manager that applies and removes flats and darks,
2313  if the task is configured to apply them.
2314 
2315  Parameters
2316  ----------
2317  exp : `lsst.afw.image.Exposure`
2318  Exposure to process.
2319  flat : `lsst.afw.image.Exposure`
2320  Flat exposure the same size as ``exp``.
2321  dark : `lsst.afw.image.Exposure`, optional
2322  Dark exposure the same size as ``exp``.
2323 
2324  Yields
2325  ------
2326  exp : `lsst.afw.image.Exposure`
2327  The flat and dark corrected exposure.
2328  """
2329  if self.config.doDark and dark is not None:
2330  self.darkCorrection(exp, dark)
2331  if self.config.doFlat:
2332  self.flatCorrection(exp, flat)
2333  try:
2334  yield exp
2335  finally:
2336  if self.config.doFlat:
2337  self.flatCorrection(exp, flat, invert=True)
2338  if self.config.doDark and dark is not None:
2339  self.darkCorrection(exp, dark, invert=True)
2340 
2341  def debugView(self, exposure, stepname):
2342  """Utility function to examine ISR exposure at different stages.
2343 
2344  Parameters
2345  ----------
2346  exposure : `lsst.afw.image.Exposure`
2347  Exposure to view.
2348  stepname : `str`
2349  State of processing to view.
2350  """
2351  frame = getDebugFrame(self._display, stepname)
2352  if frame:
2353  display = getDisplay(frame)
2354  display.scale('asinh', 'zscale')
2355  display.mtv(exposure)
2356  prompt = "Press Enter to continue [c]... "
2357  while True:
2358  ans = input(prompt).lower()
2359  if ans in ("", "c",):
2360  break
2361 
2362 
2363 class FakeAmp(object):
2364  """A Detector-like object that supports returning gain and saturation level
2365 
2366  This is used when the input exposure does not have a detector.
2367 
2368  Parameters
2369  ----------
2370  exposure : `lsst.afw.image.Exposure`
2371  Exposure to generate a fake amplifier for.
2372  config : `lsst.ip.isr.isrTaskConfig`
2373  Configuration to apply to the fake amplifier.
2374  """
2375 
2376  def __init__(self, exposure, config):
2377  self._bbox = exposure.getBBox(afwImage.LOCAL)
2379  self._gain = config.gain
2380  self._readNoise = config.readNoise
2381  self._saturation = config.saturation
2382 
2383  def getBBox(self):
2384  return self._bbox
2385 
2386  def getRawBBox(self):
2387  return self._bbox
2388 
2389  def getHasRawInfo(self):
2390  return True # but see getRawHorizontalOverscanBBox()
2391 
2393  return self._RawHorizontalOverscanBBox
2394 
2395  def getGain(self):
2396  return self._gain
2397 
2398  def getReadNoise(self):
2399  return self._readNoise
2400 
2401  def getSaturation(self):
2402  return self._saturation
2403 
2404  def getSuspectLevel(self):
2405  return float("NaN")
2406 
2407 
2408 class RunIsrConfig(pexConfig.Config):
2409  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2410 
2411 
2412 class RunIsrTask(pipeBase.CmdLineTask):
2413  """Task to wrap the default IsrTask to allow it to be retargeted.
2414 
2415  The standard IsrTask can be called directly from a command line
2416  program, but doing so removes the ability of the task to be
2417  retargeted. As most cameras override some set of the IsrTask
2418  methods, this would remove those data-specific methods in the
2419  output post-ISR images. This wrapping class fixes the issue,
2420  allowing identical post-ISR images to be generated by both the
2421  processCcd and isrTask code.
2422  """
2423  ConfigClass = RunIsrConfig
2424  _DefaultName = "runIsr"
2425 
2426  def __init__(self, *args, **kwargs):
2427  super().__init__(*args, **kwargs)
2428  self.makeSubtask("isr")
2429 
2430  def runDataRef(self, dataRef):
2431  """
2432  Parameters
2433  ----------
2434  dataRef : `lsst.daf.persistence.ButlerDataRef`
2435  data reference of the detector data to be processed
2436 
2437  Returns
2438  -------
2439  result : `pipeBase.Struct`
2440  Result struct with component:
2441 
2442  - exposure : `lsst.afw.image.Exposure`
2443  Post-ISR processed exposure.
2444  """
2445  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:765
def runDataRef(self, sensorRef)
Definition: isrTask.py:1469
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2200
def debugView(self, exposure, stepname)
Definition: isrTask.py:2341
def __init__(self, kwargs)
Definition: isrTask.py:755
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1564
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:867
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:826
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2103
def runDataRef(self, dataRef)
Definition: isrTask.py:2430
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1015
def __init__(self, args, kwargs)
Definition: isrTask.py:2426
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:817
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2264
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2129
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2392
def maskNan(self, exposure)
Definition: isrTask.py:2154
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:804
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2080
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1720
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1612
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1971
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:864
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1523
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1908
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1952
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2287
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1649
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2311
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1858
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2180
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2045
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2020
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1996
def __init__(self, exposure, config)
Definition: isrTask.py:2376