lsst.ip.isr  19.0.0-1-gf46fa72
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.PrerequisiteInput(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ImageF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ImageF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="MaskedImageF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  defects = cT.PrerequisiteInput(
112  name='defects',
113  doc="Input defect tables.",
114  storageClass="DefectsList",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  opticsTransmission = cT.PrerequisiteInput(
118  name="transmission_optics",
119  storageClass="TransmissionCurve",
120  doc="Transmission curve due to the optics.",
121  dimensions=["instrument", "calibration_label"],
122  )
123  filterTransmission = cT.PrerequisiteInput(
124  name="transmission_filter",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the filter.",
127  dimensions=["instrument", "physical_filter", "calibration_label"],
128  )
129  sensorTransmission = cT.PrerequisiteInput(
130  name="transmission_sensor",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the sensor.",
133  dimensions=["instrument", "calibration_label", "detector"],
134  )
135  atmosphereTransmission = cT.PrerequisiteInput(
136  name="transmission_atmosphere",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the atmosphere.",
139  dimensions=["instrument"],
140  )
141  illumMaskedImage = cT.PrerequisiteInput(
142  name="illum",
143  doc="Input illumination correction.",
144  storageClass="MaskedImageF",
145  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
146  )
147 
148  outputExposure = cT.Output(
149  name='postISRCCD',
150  doc="Output ISR processed exposure.",
151  storageClass="ExposureF",
152  dimensions=["instrument", "visit", "detector"],
153  )
154  preInterpExposure = cT.Output(
155  name='preInterpISRCCD',
156  doc="Output ISR processed exposure, with pixels left uninterpolated.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "visit", "detector"],
159  )
160  outputOssThumbnail = cT.Output(
161  name="OssThumb",
162  doc="Output Overscan-subtracted thumbnail image.",
163  storageClass="Thumbnail",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputFlattenedThumbnail = cT.Output(
167  name="FlattenedThumb",
168  doc="Output flat-corrected thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "visit", "detector"],
171  )
172 
173  def __init__(self, *, config=None):
174  super().__init__(config=config)
175 
176  if config.doBias is not True:
177  self.prerequisiteInputs.discard("bias")
178  if config.doLinearize is not True:
179  self.prerequisiteInputs.discard("linearizer")
180  if config.doCrosstalk is not True:
181  self.prerequisiteInputs.discard("crosstalkSources")
182  if config.doBrighterFatter is not True:
183  self.prerequisiteInputs.discard("bfKernel")
184  if config.doDefect is not True:
185  self.prerequisiteInputs.discard("defects")
186  if config.doDark is not True:
187  self.prerequisiteInputs.discard("dark")
188  if config.doFlat is not True:
189  self.prerequisiteInputs.discard("flat")
190  if config.doAttachTransmissionCurve is not True:
191  self.prerequisiteInputs.discard("opticsTransmission")
192  self.prerequisiteInputs.discard("filterTransmission")
193  self.prerequisiteInputs.discard("sensorTransmission")
194  self.prerequisiteInputs.discard("atmosphereTransmission")
195  if config.doUseOpticsTransmission is not True:
196  self.prerequisiteInputs.discard("opticsTransmission")
197  if config.doUseFilterTransmission is not True:
198  self.prerequisiteInputs.discard("filterTransmission")
199  if config.doUseSensorTransmission is not True:
200  self.prerequisiteInputs.discard("sensorTransmission")
201  if config.doUseAtmosphereTransmission is not True:
202  self.prerequisiteInputs.discard("atmosphereTransmission")
203  if config.doIlluminationCorrection is not True:
204  self.prerequisiteInputs.discard("illumMaskedImage")
205 
206  if config.doWrite is not True:
207  self.outputs.discard("outputExposure")
208  self.outputs.discard("preInterpExposure")
209  self.outputs.discard("outputFlattenedThumbnail")
210  self.outputs.discard("outputOssThumbnail")
211  if config.doSaveInterpPixels is not True:
212  self.outputs.discard("preInterpExposure")
213  if config.qa.doThumbnailOss is not True:
214  self.outputs.discard("outputOssThumbnail")
215  if config.qa.doThumbnailFlattened is not True:
216  self.outputs.discard("outputFlattenedThumbnail")
217 
218 
219 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
220  pipelineConnections=IsrTaskConnections):
221  """Configuration parameters for IsrTask.
222 
223  Items are grouped in the order in which they are executed by the task.
224  """
225  datasetType = pexConfig.Field(
226  dtype=str,
227  doc="Dataset type for input data; users will typically leave this alone, "
228  "but camera-specific ISR tasks will override it",
229  default="raw",
230  )
231 
232  fallbackFilterName = pexConfig.Field(
233  dtype=str,
234  doc="Fallback default filter name for calibrations.",
235  optional=True
236  )
237  expectWcs = pexConfig.Field(
238  dtype=bool,
239  default=True,
240  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
241  )
242  fwhm = pexConfig.Field(
243  dtype=float,
244  doc="FWHM of PSF in arcseconds.",
245  default=1.0,
246  )
247  qa = pexConfig.ConfigField(
248  dtype=isrQa.IsrQaConfig,
249  doc="QA related configuration options.",
250  )
251 
252  # Image conversion configuration
253  doConvertIntToFloat = pexConfig.Field(
254  dtype=bool,
255  doc="Convert integer raw images to floating point values?",
256  default=True,
257  )
258 
259  # Saturated pixel handling.
260  doSaturation = pexConfig.Field(
261  dtype=bool,
262  doc="Mask saturated pixels? NB: this is totally independent of the"
263  " interpolation option - this is ONLY setting the bits in the mask."
264  " To have them interpolated make sure doSaturationInterpolation=True",
265  default=True,
266  )
267  saturatedMaskName = pexConfig.Field(
268  dtype=str,
269  doc="Name of mask plane to use in saturation detection and interpolation",
270  default="SAT",
271  )
272  saturation = pexConfig.Field(
273  dtype=float,
274  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
275  default=float("NaN"),
276  )
277  growSaturationFootprintSize = pexConfig.Field(
278  dtype=int,
279  doc="Number of pixels by which to grow the saturation footprints",
280  default=1,
281  )
282 
283  # Suspect pixel handling.
284  doSuspect = pexConfig.Field(
285  dtype=bool,
286  doc="Mask suspect pixels?",
287  default=False,
288  )
289  suspectMaskName = pexConfig.Field(
290  dtype=str,
291  doc="Name of mask plane to use for suspect pixels",
292  default="SUSPECT",
293  )
294  numEdgeSuspect = pexConfig.Field(
295  dtype=int,
296  doc="Number of edge pixels to be flagged as untrustworthy.",
297  default=0,
298  )
299 
300  # Initial masking options.
301  doSetBadRegions = pexConfig.Field(
302  dtype=bool,
303  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
304  default=True,
305  )
306  badStatistic = pexConfig.ChoiceField(
307  dtype=str,
308  doc="How to estimate the average value for BAD regions.",
309  default='MEANCLIP',
310  allowed={
311  "MEANCLIP": "Correct using the (clipped) mean of good data",
312  "MEDIAN": "Correct using the median of the good data",
313  },
314  )
315 
316  # Overscan subtraction configuration.
317  doOverscan = pexConfig.Field(
318  dtype=bool,
319  doc="Do overscan subtraction?",
320  default=True,
321  )
322  overscanFitType = pexConfig.ChoiceField(
323  dtype=str,
324  doc="The method for fitting the overscan bias level.",
325  default='MEDIAN',
326  allowed={
327  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
328  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
329  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
330  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
331  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
332  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
333  "MEAN": "Correct using the mean of the overscan region",
334  "MEANCLIP": "Correct using a clipped mean of the overscan region",
335  "MEDIAN": "Correct using the median of the overscan region",
336  },
337  )
338  overscanOrder = pexConfig.Field(
339  dtype=int,
340  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
341  "or number of spline knots if overscan fit type is a spline."),
342  default=1,
343  )
344  overscanNumSigmaClip = pexConfig.Field(
345  dtype=float,
346  doc="Rejection threshold (sigma) for collapsing overscan before fit",
347  default=3.0,
348  )
349  overscanIsInt = pexConfig.Field(
350  dtype=bool,
351  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
352  default=True,
353  )
354  overscanNumLeadingColumnsToSkip = pexConfig.Field(
355  dtype=int,
356  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
357  default=0,
358  )
359  overscanNumTrailingColumnsToSkip = pexConfig.Field(
360  dtype=int,
361  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
362  default=0,
363  )
364  overscanMaxDev = pexConfig.Field(
365  dtype=float,
366  doc="Maximum deviation from the median for overscan",
367  default=1000.0, check=lambda x: x > 0
368  )
369  overscanBiasJump = pexConfig.Field(
370  dtype=bool,
371  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
372  default=False,
373  )
374  overscanBiasJumpKeyword = pexConfig.Field(
375  dtype=str,
376  doc="Header keyword containing information about devices.",
377  default="NO_SUCH_KEY",
378  )
379  overscanBiasJumpDevices = pexConfig.ListField(
380  dtype=str,
381  doc="List of devices that need piecewise overscan correction.",
382  default=(),
383  )
384  overscanBiasJumpLocation = pexConfig.Field(
385  dtype=int,
386  doc="Location of bias jump along y-axis.",
387  default=0,
388  )
389 
390  # Amplifier to CCD assembly configuration
391  doAssembleCcd = pexConfig.Field(
392  dtype=bool,
393  default=True,
394  doc="Assemble amp-level exposures into a ccd-level exposure?"
395  )
396  assembleCcd = pexConfig.ConfigurableField(
397  target=AssembleCcdTask,
398  doc="CCD assembly task",
399  )
400 
401  # General calibration configuration.
402  doAssembleIsrExposures = pexConfig.Field(
403  dtype=bool,
404  default=False,
405  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
406  )
407  doTrimToMatchCalib = pexConfig.Field(
408  dtype=bool,
409  default=False,
410  doc="Trim raw data to match calibration bounding boxes?"
411  )
412 
413  # Bias subtraction.
414  doBias = pexConfig.Field(
415  dtype=bool,
416  doc="Apply bias frame correction?",
417  default=True,
418  )
419  biasDataProductName = pexConfig.Field(
420  dtype=str,
421  doc="Name of the bias data product",
422  default="bias",
423  )
424 
425  # Variance construction
426  doVariance = pexConfig.Field(
427  dtype=bool,
428  doc="Calculate variance?",
429  default=True
430  )
431  gain = pexConfig.Field(
432  dtype=float,
433  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
434  default=float("NaN"),
435  )
436  readNoise = pexConfig.Field(
437  dtype=float,
438  doc="The read noise to use if no Detector is present in the Exposure",
439  default=0.0,
440  )
441  doEmpiricalReadNoise = pexConfig.Field(
442  dtype=bool,
443  default=False,
444  doc="Calculate empirical read noise instead of value from AmpInfo data?"
445  )
446 
447  # Linearization.
448  doLinearize = pexConfig.Field(
449  dtype=bool,
450  doc="Correct for nonlinearity of the detector's response?",
451  default=True,
452  )
453 
454  # Crosstalk.
455  doCrosstalk = pexConfig.Field(
456  dtype=bool,
457  doc="Apply intra-CCD crosstalk correction?",
458  default=False,
459  )
460  doCrosstalkBeforeAssemble = pexConfig.Field(
461  dtype=bool,
462  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
463  default=False,
464  )
465  crosstalk = pexConfig.ConfigurableField(
466  target=CrosstalkTask,
467  doc="Intra-CCD crosstalk correction",
468  )
469 
470  # Masking options.
471  doDefect = pexConfig.Field(
472  dtype=bool,
473  doc="Apply correction for CCD defects, e.g. hot pixels?",
474  default=True,
475  )
476  doNanMasking = pexConfig.Field(
477  dtype=bool,
478  doc="Mask NAN pixels?",
479  default=True,
480  )
481  doWidenSaturationTrails = pexConfig.Field(
482  dtype=bool,
483  doc="Widen bleed trails based on their width?",
484  default=True
485  )
486 
487  # Brighter-Fatter correction.
488  doBrighterFatter = pexConfig.Field(
489  dtype=bool,
490  default=False,
491  doc="Apply the brighter fatter correction"
492  )
493  brighterFatterLevel = pexConfig.ChoiceField(
494  dtype=str,
495  default="DETECTOR",
496  doc="The level at which to correct for brighter-fatter.",
497  allowed={
498  "AMP": "Every amplifier treated separately.",
499  "DETECTOR": "One kernel per detector",
500  }
501  )
502  brighterFatterKernelFile = pexConfig.Field(
503  dtype=str,
504  default='',
505  doc="Kernel file used for the brighter fatter correction"
506  )
507  brighterFatterMaxIter = pexConfig.Field(
508  dtype=int,
509  default=10,
510  doc="Maximum number of iterations for the brighter fatter correction"
511  )
512  brighterFatterThreshold = pexConfig.Field(
513  dtype=float,
514  default=1000,
515  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
516  " absolute value of the difference between the current corrected image and the one"
517  " from the previous iteration summed over all the pixels."
518  )
519  brighterFatterApplyGain = pexConfig.Field(
520  dtype=bool,
521  default=True,
522  doc="Should the gain be applied when applying the brighter fatter correction?"
523  )
524 
525  # Dark subtraction.
526  doDark = pexConfig.Field(
527  dtype=bool,
528  doc="Apply dark frame correction?",
529  default=True,
530  )
531  darkDataProductName = pexConfig.Field(
532  dtype=str,
533  doc="Name of the dark data product",
534  default="dark",
535  )
536 
537  # Camera-specific stray light removal.
538  doStrayLight = pexConfig.Field(
539  dtype=bool,
540  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
541  default=False,
542  )
543  strayLight = pexConfig.ConfigurableField(
544  target=StrayLightTask,
545  doc="y-band stray light correction"
546  )
547 
548  # Flat correction.
549  doFlat = pexConfig.Field(
550  dtype=bool,
551  doc="Apply flat field correction?",
552  default=True,
553  )
554  flatDataProductName = pexConfig.Field(
555  dtype=str,
556  doc="Name of the flat data product",
557  default="flat",
558  )
559  flatScalingType = pexConfig.ChoiceField(
560  dtype=str,
561  doc="The method for scaling the flat on the fly.",
562  default='USER',
563  allowed={
564  "USER": "Scale by flatUserScale",
565  "MEAN": "Scale by the inverse of the mean",
566  "MEDIAN": "Scale by the inverse of the median",
567  },
568  )
569  flatUserScale = pexConfig.Field(
570  dtype=float,
571  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
572  default=1.0,
573  )
574  doTweakFlat = pexConfig.Field(
575  dtype=bool,
576  doc="Tweak flats to match observed amplifier ratios?",
577  default=False
578  )
579 
580  # Amplifier normalization based on gains instead of using flats configuration.
581  doApplyGains = pexConfig.Field(
582  dtype=bool,
583  doc="Correct the amplifiers for their gains instead of applying flat correction",
584  default=False,
585  )
586  normalizeGains = pexConfig.Field(
587  dtype=bool,
588  doc="Normalize all the amplifiers in each CCD to have the same median value.",
589  default=False,
590  )
591 
592  # Fringe correction.
593  doFringe = pexConfig.Field(
594  dtype=bool,
595  doc="Apply fringe correction?",
596  default=True,
597  )
598  fringe = pexConfig.ConfigurableField(
599  target=FringeTask,
600  doc="Fringe subtraction task",
601  )
602  fringeAfterFlat = pexConfig.Field(
603  dtype=bool,
604  doc="Do fringe subtraction after flat-fielding?",
605  default=True,
606  )
607 
608  # Distortion model application.
609  doAddDistortionModel = pexConfig.Field(
610  dtype=bool,
611  doc="Apply a distortion model based on camera geometry to the WCS?",
612  default=True,
613  deprecated=("Camera geometry is incorporated when reading the raw files."
614  " This option no longer is used, and will be removed after v19.")
615  )
616 
617  # Initial CCD-level background statistics options.
618  doMeasureBackground = pexConfig.Field(
619  dtype=bool,
620  doc="Measure the background level on the reduced image?",
621  default=False,
622  )
623 
624  # Camera-specific masking configuration.
625  doCameraSpecificMasking = pexConfig.Field(
626  dtype=bool,
627  doc="Mask camera-specific bad regions?",
628  default=False,
629  )
630  masking = pexConfig.ConfigurableField(
631  target=MaskingTask,
632  doc="Masking task."
633  )
634 
635  # Interpolation options.
636 
637  doInterpolate = pexConfig.Field(
638  dtype=bool,
639  doc="Interpolate masked pixels?",
640  default=True,
641  )
642  doSaturationInterpolation = pexConfig.Field(
643  dtype=bool,
644  doc="Perform interpolation over pixels masked as saturated?"
645  " NB: This is independent of doSaturation; if that is False this plane"
646  " will likely be blank, resulting in a no-op here.",
647  default=True,
648  )
649  doNanInterpolation = pexConfig.Field(
650  dtype=bool,
651  doc="Perform interpolation over pixels masked as NaN?"
652  " NB: This is independent of doNanMasking; if that is False this plane"
653  " will likely be blank, resulting in a no-op here.",
654  default=True,
655  )
656  doNanInterpAfterFlat = pexConfig.Field(
657  dtype=bool,
658  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
659  "also have to interpolate them before flat-fielding."),
660  default=False,
661  )
662  maskListToInterpolate = pexConfig.ListField(
663  dtype=str,
664  doc="List of mask planes that should be interpolated.",
665  default=['SAT', 'BAD', 'UNMASKEDNAN'],
666  )
667  doSaveInterpPixels = pexConfig.Field(
668  dtype=bool,
669  doc="Save a copy of the pre-interpolated pixel values?",
670  default=False,
671  )
672 
673  # Default photometric calibration options.
674  fluxMag0T1 = pexConfig.DictField(
675  keytype=str,
676  itemtype=float,
677  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
678  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
679  ))
680  )
681  defaultFluxMag0T1 = pexConfig.Field(
682  dtype=float,
683  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
684  default=pow(10.0, 0.4*28.0)
685  )
686 
687  # Vignette correction configuration.
688  doVignette = pexConfig.Field(
689  dtype=bool,
690  doc="Apply vignetting parameters?",
691  default=False,
692  )
693  vignette = pexConfig.ConfigurableField(
694  target=VignetteTask,
695  doc="Vignetting task.",
696  )
697 
698  # Transmission curve configuration.
699  doAttachTransmissionCurve = pexConfig.Field(
700  dtype=bool,
701  default=False,
702  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
703  )
704  doUseOpticsTransmission = pexConfig.Field(
705  dtype=bool,
706  default=True,
707  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
708  )
709  doUseFilterTransmission = pexConfig.Field(
710  dtype=bool,
711  default=True,
712  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
713  )
714  doUseSensorTransmission = pexConfig.Field(
715  dtype=bool,
716  default=True,
717  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
718  )
719  doUseAtmosphereTransmission = pexConfig.Field(
720  dtype=bool,
721  default=True,
722  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
723  )
724 
725  # Illumination correction.
726  doIlluminationCorrection = pexConfig.Field(
727  dtype=bool,
728  default=False,
729  doc="Perform illumination correction?"
730  )
731  illuminationCorrectionDataProductName = pexConfig.Field(
732  dtype=str,
733  doc="Name of the illumination correction data product.",
734  default="illumcor",
735  )
736  illumScale = pexConfig.Field(
737  dtype=float,
738  doc="Scale factor for the illumination correction.",
739  default=1.0,
740  )
741  illumFilters = pexConfig.ListField(
742  dtype=str,
743  default=[],
744  doc="Only perform illumination correction for these filters."
745  )
746 
747  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
748  doWrite = pexConfig.Field(
749  dtype=bool,
750  doc="Persist postISRCCD?",
751  default=True,
752  )
753 
754  def validate(self):
755  super().validate()
756  if self.doFlat and self.doApplyGains:
757  raise ValueError("You may not specify both doFlat and doApplyGains")
758  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
759  self.config.maskListToInterpolate.append("SAT")
760  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
761  self.config.maskListToInterpolate.append("UNMASKEDNAN")
762 
763 
764 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
765  """Apply common instrument signature correction algorithms to a raw frame.
766 
767  The process for correcting imaging data is very similar from
768  camera to camera. This task provides a vanilla implementation of
769  doing these corrections, including the ability to turn certain
770  corrections off if they are not needed. The inputs to the primary
771  method, `run()`, are a raw exposure to be corrected and the
772  calibration data products. The raw input is a single chip sized
773  mosaic of all amps including overscans and other non-science
774  pixels. The method `runDataRef()` identifies and defines the
775  calibration data products, and is intended for use by a
776  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
777  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
778  subclassed for different camera, although the most camera specific
779  methods have been split into subtasks that can be redirected
780  appropriately.
781 
782  The __init__ method sets up the subtasks for ISR processing, using
783  the defaults from `lsst.ip.isr`.
784 
785  Parameters
786  ----------
787  args : `list`
788  Positional arguments passed to the Task constructor. None used at this time.
789  kwargs : `dict`, optional
790  Keyword arguments passed on to the Task constructor. None used at this time.
791  """
792  ConfigClass = IsrTaskConfig
793  _DefaultName = "isr"
794 
795  def __init__(self, **kwargs):
796  super().__init__(**kwargs)
797  self.makeSubtask("assembleCcd")
798  self.makeSubtask("crosstalk")
799  self.makeSubtask("strayLight")
800  self.makeSubtask("fringe")
801  self.makeSubtask("masking")
802  self.makeSubtask("vignette")
803 
804  def runQuantum(self, butlerQC, inputRefs, outputRefs):
805  inputs = butlerQC.get(inputRefs)
806 
807  try:
808  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
809  except Exception as e:
810  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
811  (inputRefs, e))
812 
813  inputs['isGen3'] = True
814 
815  if self.config.doLinearize is True:
816  if 'linearizer' not in inputs:
817  detector = inputs['ccdExposure'].getDetector()
818  linearityName = detector.getAmplifiers()[0].getLinearityType()
819  inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
820 
821  if self.config.doDefect is True:
822  if "defects" in inputs and inputs['defects'] is not None:
823  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
824  # masking expects a list of defects defined by their bounding box
825  if not isinstance(inputs["defects"], Defects):
826  inputs["defects"] = Defects.fromTable(inputs["defects"])
827 
828  # Broken: DM-17169
829  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
830  # fixed for non-HSC cameras in the future.
831  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
832  # if self.config.doCrosstalk else None)
833 
834  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
835  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
836  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
837  expId=expId,
838  assembler=self.assembleCcd
839  if self.config.doAssembleIsrExposures else None)
840  else:
841  inputs['fringes'] = pipeBase.Struct(fringes=None)
842 
843  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
844  if 'strayLightData' not in inputs:
845  inputs['strayLightData'] = None
846 
847  outputs = self.run(**inputs)
848  butlerQC.put(outputs, outputRefs)
849 
850  def readIsrData(self, dataRef, rawExposure):
851  """!Retrieve necessary frames for instrument signature removal.
852 
853  Pre-fetching all required ISR data products limits the IO
854  required by the ISR. Any conflict between the calibration data
855  available and that needed for ISR is also detected prior to
856  doing processing, allowing it to fail quickly.
857 
858  Parameters
859  ----------
860  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
861  Butler reference of the detector data to be processed
862  rawExposure : `afw.image.Exposure`
863  The raw exposure that will later be corrected with the
864  retrieved calibration data; should not be modified in this
865  method.
866 
867  Returns
868  -------
869  result : `lsst.pipe.base.Struct`
870  Result struct with components (which may be `None`):
871  - ``bias``: bias calibration frame (`afw.image.Exposure`)
872  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
873  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
874  - ``dark``: dark calibration frame (`afw.image.Exposure`)
875  - ``flat``: flat calibration frame (`afw.image.Exposure`)
876  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
877  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
878  - ``fringes``: `lsst.pipe.base.Struct` with components:
879  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
880  - ``seed``: random seed derived from the ccdExposureId for random
881  number generator (`uint32`).
882  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
883  A ``TransmissionCurve`` that represents the throughput of the optics,
884  to be evaluated in focal-plane coordinates.
885  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
886  A ``TransmissionCurve`` that represents the throughput of the filter
887  itself, to be evaluated in focal-plane coordinates.
888  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
889  A ``TransmissionCurve`` that represents the throughput of the sensor
890  itself, to be evaluated in post-assembly trimmed detector coordinates.
891  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
892  A ``TransmissionCurve`` that represents the throughput of the
893  atmosphere, assumed to be spatially constant.
894  - ``strayLightData`` : `object`
895  An opaque object containing calibration information for
896  stray-light correction. If `None`, no correction will be
897  performed.
898  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
899 
900  Raises
901  ------
902  NotImplementedError :
903  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
904  """
905  ccd = rawExposure.getDetector()
906  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
907  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
908  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
909  if self.config.doBias else None)
910  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
911  linearizer = (dataRef.get("linearizer", immediate=True)
912  if self.doLinearize(ccd) else None)
913  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
914  if self.config.doCrosstalk else None)
915  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
916  if self.config.doDark else None)
917  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
918  if self.config.doFlat else None)
919 
920  brighterFatterKernel = None
921  if self.config.doBrighterFatter is True:
922 
923  # Use the new-style cp_pipe version of the kernel is it exists.
924  try:
925  brighterFatterKernel = dataRef.get("brighterFatterKernel")
926  except NoResults:
927  # Fall back to the old-style numpy-ndarray style kernel if necessary.
928  try:
929  brighterFatterKernel = dataRef.get("bfKernel")
930  except NoResults:
931  brighterFatterKernel = None
932  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
933  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
934  # this detector, or raise an error.
935  if self.config.brighterFatterLevel == 'DETECTOR':
936  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
937  else:
938  # TODO DM-15631 for implementing this
939  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
940 
941  defectList = (dataRef.get("defects")
942  if self.config.doDefect else None)
943  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
944  if self.config.doAssembleIsrExposures else None)
945  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
946  else pipeBase.Struct(fringes=None))
947 
948  if self.config.doAttachTransmissionCurve:
949  opticsTransmission = (dataRef.get("transmission_optics")
950  if self.config.doUseOpticsTransmission else None)
951  filterTransmission = (dataRef.get("transmission_filter")
952  if self.config.doUseFilterTransmission else None)
953  sensorTransmission = (dataRef.get("transmission_sensor")
954  if self.config.doUseSensorTransmission else None)
955  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
956  if self.config.doUseAtmosphereTransmission else None)
957  else:
958  opticsTransmission = None
959  filterTransmission = None
960  sensorTransmission = None
961  atmosphereTransmission = None
962 
963  if self.config.doStrayLight:
964  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
965  else:
966  strayLightData = None
967 
968  illumMaskedImage = (self.getIsrExposure(dataRef,
969  self.config.illuminationCorrectionDataProductName).getMaskedImage()
970  if (self.config.doIlluminationCorrection and
971  filterName in self.config.illumFilters)
972  else None)
973 
974  # Struct should include only kwargs to run()
975  return pipeBase.Struct(bias=biasExposure,
976  linearizer=linearizer,
977  crosstalkSources=crosstalkSources,
978  dark=darkExposure,
979  flat=flatExposure,
980  bfKernel=brighterFatterKernel,
981  defects=defectList,
982  fringes=fringeStruct,
983  opticsTransmission=opticsTransmission,
984  filterTransmission=filterTransmission,
985  sensorTransmission=sensorTransmission,
986  atmosphereTransmission=atmosphereTransmission,
987  strayLightData=strayLightData,
988  illumMaskedImage=illumMaskedImage
989  )
990 
991  @pipeBase.timeMethod
992  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
993  dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
994  opticsTransmission=None, filterTransmission=None,
995  sensorTransmission=None, atmosphereTransmission=None,
996  detectorNum=None, strayLightData=None, illumMaskedImage=None,
997  isGen3=False,
998  ):
999  """!Perform instrument signature removal on an exposure.
1000 
1001  Steps included in the ISR processing, in order performed, are:
1002  - saturation and suspect pixel masking
1003  - overscan subtraction
1004  - CCD assembly of individual amplifiers
1005  - bias subtraction
1006  - variance image construction
1007  - linearization of non-linear response
1008  - crosstalk masking
1009  - brighter-fatter correction
1010  - dark subtraction
1011  - fringe correction
1012  - stray light subtraction
1013  - flat correction
1014  - masking of known defects and camera specific features
1015  - vignette calculation
1016  - appending transmission curve and distortion model
1017 
1018  Parameters
1019  ----------
1020  ccdExposure : `lsst.afw.image.Exposure`
1021  The raw exposure that is to be run through ISR. The
1022  exposure is modified by this method.
1023  camera : `lsst.afw.cameraGeom.Camera`, optional
1024  The camera geometry for this exposure. Used to select the
1025  distortion model appropriate for this data.
1026  bias : `lsst.afw.image.Exposure`, optional
1027  Bias calibration frame.
1028  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1029  Functor for linearization.
1030  crosstalkSources : `list`, optional
1031  List of possible crosstalk sources.
1032  dark : `lsst.afw.image.Exposure`, optional
1033  Dark calibration frame.
1034  flat : `lsst.afw.image.Exposure`, optional
1035  Flat calibration frame.
1036  bfKernel : `numpy.ndarray`, optional
1037  Brighter-fatter kernel.
1038  defects : `lsst.meas.algorithms.Defects`, optional
1039  List of defects.
1040  fringes : `lsst.pipe.base.Struct`, optional
1041  Struct containing the fringe correction data, with
1042  elements:
1043  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1044  - ``seed``: random seed derived from the ccdExposureId for random
1045  number generator (`uint32`)
1046  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1047  A ``TransmissionCurve`` that represents the throughput of the optics,
1048  to be evaluated in focal-plane coordinates.
1049  filterTransmission : `lsst.afw.image.TransmissionCurve`
1050  A ``TransmissionCurve`` that represents the throughput of the filter
1051  itself, to be evaluated in focal-plane coordinates.
1052  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1053  A ``TransmissionCurve`` that represents the throughput of the sensor
1054  itself, to be evaluated in post-assembly trimmed detector coordinates.
1055  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1056  A ``TransmissionCurve`` that represents the throughput of the
1057  atmosphere, assumed to be spatially constant.
1058  detectorNum : `int`, optional
1059  The integer number for the detector to process.
1060  isGen3 : bool, optional
1061  Flag this call to run() as using the Gen3 butler environment.
1062  strayLightData : `object`, optional
1063  Opaque object containing calibration information for stray-light
1064  correction. If `None`, no correction will be performed.
1065  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1066  Illumination correction image.
1067 
1068  Returns
1069  -------
1070  result : `lsst.pipe.base.Struct`
1071  Result struct with component:
1072  - ``exposure`` : `afw.image.Exposure`
1073  The fully ISR corrected exposure.
1074  - ``outputExposure`` : `afw.image.Exposure`
1075  An alias for `exposure`
1076  - ``ossThumb`` : `numpy.ndarray`
1077  Thumbnail image of the exposure after overscan subtraction.
1078  - ``flattenedThumb`` : `numpy.ndarray`
1079  Thumbnail image of the exposure after flat-field correction.
1080 
1081  Raises
1082  ------
1083  RuntimeError
1084  Raised if a configuration option is set to True, but the
1085  required calibration data has not been specified.
1086 
1087  Notes
1088  -----
1089  The current processed exposure can be viewed by setting the
1090  appropriate lsstDebug entries in the `debug.display`
1091  dictionary. The names of these entries correspond to some of
1092  the IsrTaskConfig Boolean options, with the value denoting the
1093  frame to use. The exposure is shown inside the matching
1094  option check and after the processing of that step has
1095  finished. The steps with debug points are:
1096 
1097  doAssembleCcd
1098  doBias
1099  doCrosstalk
1100  doBrighterFatter
1101  doDark
1102  doFringe
1103  doStrayLight
1104  doFlat
1105 
1106  In addition, setting the "postISRCCD" entry displays the
1107  exposure after all ISR processing has finished.
1108 
1109  """
1110 
1111  if isGen3 is True:
1112  # Gen3 currently cannot automatically do configuration overrides.
1113  # DM-15257 looks to discuss this issue.
1114  # Configure input exposures;
1115  if detectorNum is None:
1116  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1117 
1118  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1119  bias = self.ensureExposure(bias, camera, detectorNum)
1120  dark = self.ensureExposure(dark, camera, detectorNum)
1121  flat = self.ensureExposure(flat, camera, detectorNum)
1122  else:
1123  if isinstance(ccdExposure, ButlerDataRef):
1124  return self.runDataRef(ccdExposure)
1125 
1126  ccd = ccdExposure.getDetector()
1127  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1128 
1129  if not ccd:
1130  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1131  ccd = [FakeAmp(ccdExposure, self.config)]
1132 
1133  # Validate Input
1134  if self.config.doBias and bias is None:
1135  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1136  if self.doLinearize(ccd) and linearizer is None:
1137  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1138  if self.config.doBrighterFatter and bfKernel is None:
1139  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1140  if self.config.doDark and dark is None:
1141  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1142  if self.config.doFlat and flat is None:
1143  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1144  if self.config.doDefect and defects is None:
1145  raise RuntimeError("Must supply defects if config.doDefect=True.")
1146  if (self.config.doFringe and filterName in self.fringe.config.filters and
1147  fringes.fringes is None):
1148  # The `fringes` object needs to be a pipeBase.Struct, as
1149  # we use it as a `dict` for the parameters of
1150  # `FringeTask.run()`. The `fringes.fringes` `list` may
1151  # not be `None` if `doFringe=True`. Otherwise, raise.
1152  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1153  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1154  illumMaskedImage is None):
1155  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1156 
1157  # Begin ISR processing.
1158  if self.config.doConvertIntToFloat:
1159  self.log.info("Converting exposure to floating point values.")
1160  ccdExposure = self.convertIntToFloat(ccdExposure)
1161 
1162  # Amplifier level processing.
1163  overscans = []
1164  for amp in ccd:
1165  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1166  if ccdExposure.getBBox().contains(amp.getBBox()):
1167  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1168  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1169 
1170  if self.config.doOverscan and not badAmp:
1171  # Overscan correction on amp-by-amp basis.
1172  overscanResults = self.overscanCorrection(ccdExposure, amp)
1173  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1174  if overscanResults is not None and \
1175  self.config.qa is not None and self.config.qa.saveStats is True:
1176  if isinstance(overscanResults.overscanFit, float):
1177  qaMedian = overscanResults.overscanFit
1178  qaStdev = float("NaN")
1179  else:
1180  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1181  afwMath.MEDIAN | afwMath.STDEVCLIP)
1182  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1183  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1184 
1185  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1186  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1187  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1188  amp.getName(), qaMedian, qaStdev)
1189  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1190  else:
1191  if badAmp:
1192  self.log.warn("Amplifier %s is bad.", amp.getName())
1193  overscanResults = None
1194 
1195  overscans.append(overscanResults if overscanResults is not None else None)
1196  else:
1197  self.log.info("Skipped OSCAN for %s.", amp.getName())
1198 
1199  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1200  self.log.info("Applying crosstalk correction.")
1201  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1202  self.debugView(ccdExposure, "doCrosstalk")
1203 
1204  if self.config.doAssembleCcd:
1205  self.log.info("Assembling CCD from amplifiers.")
1206  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1207 
1208  if self.config.expectWcs and not ccdExposure.getWcs():
1209  self.log.warn("No WCS found in input exposure.")
1210  self.debugView(ccdExposure, "doAssembleCcd")
1211 
1212  ossThumb = None
1213  if self.config.qa.doThumbnailOss:
1214  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1215 
1216  if self.config.doBias:
1217  self.log.info("Applying bias correction.")
1218  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1219  trimToFit=self.config.doTrimToMatchCalib)
1220  self.debugView(ccdExposure, "doBias")
1221 
1222  if self.config.doVariance:
1223  for amp, overscanResults in zip(ccd, overscans):
1224  if ccdExposure.getBBox().contains(amp.getBBox()):
1225  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1226  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1227  if overscanResults is not None:
1228  self.updateVariance(ampExposure, amp,
1229  overscanImage=overscanResults.overscanImage)
1230  else:
1231  self.updateVariance(ampExposure, amp,
1232  overscanImage=None)
1233  if self.config.qa is not None and self.config.qa.saveStats is True:
1234  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1235  afwMath.MEDIAN | afwMath.STDEVCLIP)
1236  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1237  qaStats.getValue(afwMath.MEDIAN))
1238  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1239  qaStats.getValue(afwMath.STDEVCLIP))
1240  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1241  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1242  qaStats.getValue(afwMath.STDEVCLIP))
1243 
1244  if self.doLinearize(ccd):
1245  self.log.info("Applying linearizer.")
1246  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1247 
1248  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1249  self.log.info("Applying crosstalk correction.")
1250  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1251  self.debugView(ccdExposure, "doCrosstalk")
1252 
1253  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1254  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1255  if self.config.doDefect:
1256  self.log.info("Masking defects.")
1257  self.maskDefect(ccdExposure, defects)
1258 
1259  if self.config.numEdgeSuspect > 0:
1260  self.log.info("Masking edges as SUSPECT.")
1261  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1262  maskPlane="SUSPECT")
1263 
1264  if self.config.doNanMasking:
1265  self.log.info("Masking NAN value pixels.")
1266  self.maskNan(ccdExposure)
1267 
1268  if self.config.doWidenSaturationTrails:
1269  self.log.info("Widening saturation trails.")
1270  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1271 
1272  if self.config.doCameraSpecificMasking:
1273  self.log.info("Masking regions for camera specific reasons.")
1274  self.masking.run(ccdExposure)
1275 
1276  if self.config.doBrighterFatter:
1277  # We need to apply flats and darks before we can interpolate, and we
1278  # need to interpolate before we do B-F, but we do B-F without the
1279  # flats and darks applied so we can work in units of electrons or holes.
1280  # This context manager applies and then removes the darks and flats.
1281  #
1282  # We also do not want to interpolate values here, so operate on temporary
1283  # images so we can apply only the BF-correction and roll back the
1284  # interpolation.
1285  interpExp = ccdExposure.clone()
1286  with self.flatContext(interpExp, flat, dark):
1287  isrFunctions.interpolateFromMask(
1288  maskedImage=interpExp.getMaskedImage(),
1289  fwhm=self.config.fwhm,
1290  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1291  maskNameList=self.config.maskListToInterpolate
1292  )
1293  bfExp = interpExp.clone()
1294 
1295  self.log.info("Applying brighter fatter correction.")
1296  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1297  self.config.brighterFatterMaxIter,
1298  self.config.brighterFatterThreshold,
1299  self.config.brighterFatterApplyGain
1300  )
1301  if bfResults[1] == self.config.brighterFatterMaxIter:
1302  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1303  bfResults[0])
1304  else:
1305  self.log.info("Finished brighter fatter correction in %d iterations.",
1306  bfResults[1])
1307  image = ccdExposure.getMaskedImage().getImage()
1308  bfCorr = bfExp.getMaskedImage().getImage()
1309  bfCorr -= interpExp.getMaskedImage().getImage()
1310  image += bfCorr
1311 
1312  # Applying the brighter-fatter correction applies a
1313  # convolution to the science image. At the edges this
1314  # convolution may not have sufficient valid pixels to
1315  # produce a valid correction. Mark pixels within the size
1316  # of the brighter-fatter kernel as EDGE to warn of this
1317  # fact.
1318  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1319  maskPlane="EDGE")
1320  self.log.warn("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1321 
1322  self.debugView(ccdExposure, "doBrighterFatter")
1323 
1324  if self.config.doDark:
1325  self.log.info("Applying dark correction.")
1326  self.darkCorrection(ccdExposure, dark)
1327  self.debugView(ccdExposure, "doDark")
1328 
1329  if self.config.doFringe and not self.config.fringeAfterFlat:
1330  self.log.info("Applying fringe correction before flat.")
1331  self.fringe.run(ccdExposure, **fringes.getDict())
1332  self.debugView(ccdExposure, "doFringe")
1333 
1334  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1335  self.log.info("Checking strayLight correction.")
1336  self.strayLight.run(ccdExposure, strayLightData)
1337  self.debugView(ccdExposure, "doStrayLight")
1338 
1339  if self.config.doFlat:
1340  self.log.info("Applying flat correction.")
1341  self.flatCorrection(ccdExposure, flat)
1342  self.debugView(ccdExposure, "doFlat")
1343 
1344  if self.config.doApplyGains:
1345  self.log.info("Applying gain correction instead of flat.")
1346  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1347 
1348  if self.config.doFringe and self.config.fringeAfterFlat:
1349  self.log.info("Applying fringe correction after flat.")
1350  self.fringe.run(ccdExposure, **fringes.getDict())
1351 
1352  if self.config.doVignette:
1353  self.log.info("Constructing Vignette polygon.")
1354  self.vignettePolygon = self.vignette.run(ccdExposure)
1355 
1356  if self.config.vignette.doWriteVignettePolygon:
1357  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1358 
1359  if self.config.doAttachTransmissionCurve:
1360  self.log.info("Adding transmission curves.")
1361  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1362  filterTransmission=filterTransmission,
1363  sensorTransmission=sensorTransmission,
1364  atmosphereTransmission=atmosphereTransmission)
1365 
1366  flattenedThumb = None
1367  if self.config.qa.doThumbnailFlattened:
1368  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1369 
1370  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1371  self.log.info("Performing illumination correction.")
1372  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1373  illumMaskedImage, illumScale=self.config.illumScale,
1374  trimToFit=self.config.doTrimToMatchCalib)
1375 
1376  preInterpExp = None
1377  if self.config.doSaveInterpPixels:
1378  preInterpExp = ccdExposure.clone()
1379 
1380  # Reset and interpolate bad pixels.
1381  #
1382  # Large contiguous bad regions (which should have the BAD mask
1383  # bit set) should have their values set to the image median.
1384  # This group should include defects and bad amplifiers. As the
1385  # area covered by these defects are large, there's little
1386  # reason to expect that interpolation would provide a more
1387  # useful value.
1388  #
1389  # Smaller defects can be safely interpolated after the larger
1390  # regions have had their pixel values reset. This ensures
1391  # that the remaining defects adjacent to bad amplifiers (as an
1392  # example) do not attempt to interpolate extreme values.
1393  if self.config.doSetBadRegions:
1394  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1395  if badPixelCount > 0:
1396  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1397 
1398  if self.config.doInterpolate:
1399  self.log.info("Interpolating masked pixels.")
1400  isrFunctions.interpolateFromMask(
1401  maskedImage=ccdExposure.getMaskedImage(),
1402  fwhm=self.config.fwhm,
1403  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1404  maskNameList=list(self.config.maskListToInterpolate)
1405  )
1406 
1407  self.roughZeroPoint(ccdExposure)
1408 
1409  if self.config.doMeasureBackground:
1410  self.log.info("Measuring background level.")
1411  self.measureBackground(ccdExposure, self.config.qa)
1412 
1413  if self.config.qa is not None and self.config.qa.saveStats is True:
1414  for amp in ccd:
1415  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1416  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1417  afwMath.MEDIAN | afwMath.STDEVCLIP)
1418  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1419  qaStats.getValue(afwMath.MEDIAN))
1420  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1421  qaStats.getValue(afwMath.STDEVCLIP))
1422  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1423  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1424  qaStats.getValue(afwMath.STDEVCLIP))
1425 
1426  self.debugView(ccdExposure, "postISRCCD")
1427 
1428  return pipeBase.Struct(
1429  exposure=ccdExposure,
1430  ossThumb=ossThumb,
1431  flattenedThumb=flattenedThumb,
1432 
1433  preInterpolatedExposure=preInterpExp,
1434  outputExposure=ccdExposure,
1435  outputOssThumbnail=ossThumb,
1436  outputFlattenedThumbnail=flattenedThumb,
1437  )
1438 
1439  @pipeBase.timeMethod
1440  def runDataRef(self, sensorRef):
1441  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1442 
1443  This method contains the `CmdLineTask` interface to the ISR
1444  processing. All IO is handled here, freeing the `run()` method
1445  to manage only pixel-level calculations. The steps performed
1446  are:
1447  - Read in necessary detrending/isr/calibration data.
1448  - Process raw exposure in `run()`.
1449  - Persist the ISR-corrected exposure as "postISRCCD" if
1450  config.doWrite=True.
1451 
1452  Parameters
1453  ----------
1454  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1455  DataRef of the detector data to be processed
1456 
1457  Returns
1458  -------
1459  result : `lsst.pipe.base.Struct`
1460  Result struct with component:
1461  - ``exposure`` : `afw.image.Exposure`
1462  The fully ISR corrected exposure.
1463 
1464  Raises
1465  ------
1466  RuntimeError
1467  Raised if a configuration option is set to True, but the
1468  required calibration data does not exist.
1469 
1470  """
1471  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1472 
1473  ccdExposure = sensorRef.get(self.config.datasetType)
1474 
1475  camera = sensorRef.get("camera")
1476  isrData = self.readIsrData(sensorRef, ccdExposure)
1477 
1478  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1479 
1480  if self.config.doWrite:
1481  sensorRef.put(result.exposure, "postISRCCD")
1482  if result.preInterpolatedExposure is not None:
1483  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1484  if result.ossThumb is not None:
1485  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1486  if result.flattenedThumb is not None:
1487  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1488 
1489  return result
1490 
1491  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1492  """!Retrieve a calibration dataset for removing instrument signature.
1493 
1494  Parameters
1495  ----------
1496 
1497  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1498  DataRef of the detector data to find calibration datasets
1499  for.
1500  datasetType : `str`
1501  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1502  immediate : `Bool`
1503  If True, disable butler proxies to enable error handling
1504  within this routine.
1505 
1506  Returns
1507  -------
1508  exposure : `lsst.afw.image.Exposure`
1509  Requested calibration frame.
1510 
1511  Raises
1512  ------
1513  RuntimeError
1514  Raised if no matching calibration frame can be found.
1515  """
1516  try:
1517  exp = dataRef.get(datasetType, immediate=immediate)
1518  except Exception as exc1:
1519  if not self.config.fallbackFilterName:
1520  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1521  try:
1522  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1523  except Exception as exc2:
1524  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1525  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1526  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1527 
1528  if self.config.doAssembleIsrExposures:
1529  exp = self.assembleCcd.assembleCcd(exp)
1530  return exp
1531 
1532  def ensureExposure(self, inputExp, camera, detectorNum):
1533  """Ensure that the data returned by Butler is a fully constructed exposure.
1534 
1535  ISR requires exposure-level image data for historical reasons, so if we did
1536  not recieve that from Butler, construct it from what we have, modifying the
1537  input in place.
1538 
1539  Parameters
1540  ----------
1541  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1542  `lsst.afw.image.ImageF`
1543  The input data structure obtained from Butler.
1544  camera : `lsst.afw.cameraGeom.camera`
1545  The camera associated with the image. Used to find the appropriate
1546  detector.
1547  detectorNum : `int`
1548  The detector this exposure should match.
1549 
1550  Returns
1551  -------
1552  inputExp : `lsst.afw.image.Exposure`
1553  The re-constructed exposure, with appropriate detector parameters.
1554 
1555  Raises
1556  ------
1557  TypeError
1558  Raised if the input data cannot be used to construct an exposure.
1559  """
1560  if isinstance(inputExp, afwImage.DecoratedImageU):
1561  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1562  elif isinstance(inputExp, afwImage.ImageF):
1563  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1564  elif isinstance(inputExp, afwImage.MaskedImageF):
1565  inputExp = afwImage.makeExposure(inputExp)
1566  elif isinstance(inputExp, afwImage.Exposure):
1567  pass
1568  elif inputExp is None:
1569  # Assume this will be caught by the setup if it is a problem.
1570  return inputExp
1571  else:
1572  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1573  (type(inputExp), ))
1574 
1575  if inputExp.getDetector() is None:
1576  inputExp.setDetector(camera[detectorNum])
1577 
1578  return inputExp
1579 
1580  def convertIntToFloat(self, exposure):
1581  """Convert exposure image from uint16 to float.
1582 
1583  If the exposure does not need to be converted, the input is
1584  immediately returned. For exposures that are converted to use
1585  floating point pixels, the variance is set to unity and the
1586  mask to zero.
1587 
1588  Parameters
1589  ----------
1590  exposure : `lsst.afw.image.Exposure`
1591  The raw exposure to be converted.
1592 
1593  Returns
1594  -------
1595  newexposure : `lsst.afw.image.Exposure`
1596  The input ``exposure``, converted to floating point pixels.
1597 
1598  Raises
1599  ------
1600  RuntimeError
1601  Raised if the exposure type cannot be converted to float.
1602 
1603  """
1604  if isinstance(exposure, afwImage.ExposureF):
1605  # Nothing to be done
1606  self.log.debug("Exposure already of type float.")
1607  return exposure
1608  if not hasattr(exposure, "convertF"):
1609  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1610 
1611  newexposure = exposure.convertF()
1612  newexposure.variance[:] = 1
1613  newexposure.mask[:] = 0x0
1614 
1615  return newexposure
1616 
1617  def maskAmplifier(self, ccdExposure, amp, defects):
1618  """Identify bad amplifiers, saturated and suspect pixels.
1619 
1620  Parameters
1621  ----------
1622  ccdExposure : `lsst.afw.image.Exposure`
1623  Input exposure to be masked.
1624  amp : `lsst.afw.table.AmpInfoCatalog`
1625  Catalog of parameters defining the amplifier on this
1626  exposure to mask.
1627  defects : `lsst.meas.algorithms.Defects`
1628  List of defects. Used to determine if the entire
1629  amplifier is bad.
1630 
1631  Returns
1632  -------
1633  badAmp : `Bool`
1634  If this is true, the entire amplifier area is covered by
1635  defects and unusable.
1636 
1637  """
1638  maskedImage = ccdExposure.getMaskedImage()
1639 
1640  badAmp = False
1641 
1642  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1643  # comparison with current defects definition.
1644  if defects is not None:
1645  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1646 
1647  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1648  # association with pixels in current ccdExposure).
1649  if badAmp:
1650  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1651  afwImage.PARENT)
1652  maskView = dataView.getMask()
1653  maskView |= maskView.getPlaneBitMask("BAD")
1654  del maskView
1655  return badAmp
1656 
1657  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1658  # Saturation and suspect pixels can be masked now, though.
1659  limits = dict()
1660  if self.config.doSaturation and not badAmp:
1661  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1662  if self.config.doSuspect and not badAmp:
1663  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1664  if math.isfinite(self.config.saturation):
1665  limits.update({self.config.saturatedMaskName: self.config.saturation})
1666 
1667  for maskName, maskThreshold in limits.items():
1668  if not math.isnan(maskThreshold):
1669  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1670  isrFunctions.makeThresholdMask(
1671  maskedImage=dataView,
1672  threshold=maskThreshold,
1673  growFootprints=0,
1674  maskName=maskName
1675  )
1676 
1677  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1678  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1679  afwImage.PARENT)
1680  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1681  self.config.suspectMaskName])
1682  if numpy.all(maskView.getArray() & maskVal > 0):
1683  badAmp = True
1684  maskView |= maskView.getPlaneBitMask("BAD")
1685 
1686  return badAmp
1687 
1688  def overscanCorrection(self, ccdExposure, amp):
1689  """Apply overscan correction in place.
1690 
1691  This method does initial pixel rejection of the overscan
1692  region. The overscan can also be optionally segmented to
1693  allow for discontinuous overscan responses to be fit
1694  separately. The actual overscan subtraction is performed by
1695  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1696  which is called here after the amplifier is preprocessed.
1697 
1698  Parameters
1699  ----------
1700  ccdExposure : `lsst.afw.image.Exposure`
1701  Exposure to have overscan correction performed.
1702  amp : `lsst.afw.table.AmpInfoCatalog`
1703  The amplifier to consider while correcting the overscan.
1704 
1705  Returns
1706  -------
1707  overscanResults : `lsst.pipe.base.Struct`
1708  Result struct with components:
1709  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1710  Value or fit subtracted from the amplifier image data.
1711  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1712  Value or fit subtracted from the overscan image data.
1713  - ``overscanImage`` : `lsst.afw.image.Image`
1714  Image of the overscan region with the overscan
1715  correction applied. This quantity is used to estimate
1716  the amplifier read noise empirically.
1717 
1718  Raises
1719  ------
1720  RuntimeError
1721  Raised if the ``amp`` does not contain raw pixel information.
1722 
1723  See Also
1724  --------
1725  lsst.ip.isr.isrFunctions.overscanCorrection
1726  """
1727  if not amp.getHasRawInfo():
1728  raise RuntimeError("This method must be executed on an amp with raw information.")
1729 
1730  if amp.getRawHorizontalOverscanBBox().isEmpty():
1731  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1732  return None
1733 
1734  statControl = afwMath.StatisticsControl()
1735  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1736 
1737  # Determine the bounding boxes
1738  dataBBox = amp.getRawDataBBox()
1739  oscanBBox = amp.getRawHorizontalOverscanBBox()
1740  dx0 = 0
1741  dx1 = 0
1742 
1743  prescanBBox = amp.getRawPrescanBBox()
1744  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1745  dx0 += self.config.overscanNumLeadingColumnsToSkip
1746  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1747  else:
1748  dx0 += self.config.overscanNumTrailingColumnsToSkip
1749  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1750 
1751  # Determine if we need to work on subregions of the amplifier and overscan.
1752  imageBBoxes = []
1753  overscanBBoxes = []
1754 
1755  if ((self.config.overscanBiasJump and
1756  self.config.overscanBiasJumpLocation) and
1757  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1758  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1759  self.config.overscanBiasJumpDevices)):
1760  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1761  yLower = self.config.overscanBiasJumpLocation
1762  yUpper = dataBBox.getHeight() - yLower
1763  else:
1764  yUpper = self.config.overscanBiasJumpLocation
1765  yLower = dataBBox.getHeight() - yUpper
1766 
1767  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1768  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1769  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1770  lsst.geom.Extent2I(dx0, 0),
1771  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1772  yLower)))
1773 
1774  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1775  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1776  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1777  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1778  yUpper)))
1779  else:
1780  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1781  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1782  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1783  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1784  oscanBBox.getHeight())))
1785 
1786  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1787  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1788  ampImage = ccdExposure.maskedImage[imageBBox]
1789  overscanImage = ccdExposure.maskedImage[overscanBBox]
1790 
1791  overscanArray = overscanImage.image.array
1792  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1793  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1794  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1795 
1796  statControl = afwMath.StatisticsControl()
1797  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1798 
1799  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1800  overscanImage=overscanImage,
1801  fitType=self.config.overscanFitType,
1802  order=self.config.overscanOrder,
1803  collapseRej=self.config.overscanNumSigmaClip,
1804  statControl=statControl,
1805  overscanIsInt=self.config.overscanIsInt
1806  )
1807 
1808  # Measure average overscan levels and record them in the metadata.
1809  levelStat = afwMath.MEDIAN
1810  sigmaStat = afwMath.STDEVCLIP
1811 
1812  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1813  self.config.qa.flatness.nIter)
1814  metadata = ccdExposure.getMetadata()
1815  ampNum = amp.getName()
1816  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1817  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1818  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1819  else:
1820  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1821  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1822  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1823 
1824  return overscanResults
1825 
1826  def updateVariance(self, ampExposure, amp, overscanImage=None):
1827  """Set the variance plane using the amplifier gain and read noise
1828 
1829  The read noise is calculated from the ``overscanImage`` if the
1830  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1831  the value from the amplifier data is used.
1832 
1833  Parameters
1834  ----------
1835  ampExposure : `lsst.afw.image.Exposure`
1836  Exposure to process.
1837  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1838  Amplifier detector data.
1839  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1840  Image of overscan, required only for empirical read noise.
1841 
1842  See also
1843  --------
1844  lsst.ip.isr.isrFunctions.updateVariance
1845  """
1846  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1847  gain = amp.getGain()
1848 
1849  if math.isnan(gain):
1850  gain = 1.0
1851  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1852  elif gain <= 0:
1853  patchedGain = 1.0
1854  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1855  amp.getName(), gain, patchedGain)
1856  gain = patchedGain
1857 
1858  if self.config.doEmpiricalReadNoise and overscanImage is None:
1859  self.log.info("Overscan is none for EmpiricalReadNoise.")
1860 
1861  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1862  stats = afwMath.StatisticsControl()
1863  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1864  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1865  self.log.info("Calculated empirical read noise for amp %s: %f.",
1866  amp.getName(), readNoise)
1867  else:
1868  readNoise = amp.getReadNoise()
1869 
1870  isrFunctions.updateVariance(
1871  maskedImage=ampExposure.getMaskedImage(),
1872  gain=gain,
1873  readNoise=readNoise,
1874  )
1875 
1876  def darkCorrection(self, exposure, darkExposure, invert=False):
1877  """!Apply dark correction in place.
1878 
1879  Parameters
1880  ----------
1881  exposure : `lsst.afw.image.Exposure`
1882  Exposure to process.
1883  darkExposure : `lsst.afw.image.Exposure`
1884  Dark exposure of the same size as ``exposure``.
1885  invert : `Bool`, optional
1886  If True, re-add the dark to an already corrected image.
1887 
1888  Raises
1889  ------
1890  RuntimeError
1891  Raised if either ``exposure`` or ``darkExposure`` do not
1892  have their dark time defined.
1893 
1894  See Also
1895  --------
1896  lsst.ip.isr.isrFunctions.darkCorrection
1897  """
1898  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1899  if math.isnan(expScale):
1900  raise RuntimeError("Exposure darktime is NAN.")
1901  if darkExposure.getInfo().getVisitInfo() is not None:
1902  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1903  else:
1904  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1905  # so getDarkTime() does not exist.
1906  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1907  darkScale = 1.0
1908 
1909  if math.isnan(darkScale):
1910  raise RuntimeError("Dark calib darktime is NAN.")
1911  isrFunctions.darkCorrection(
1912  maskedImage=exposure.getMaskedImage(),
1913  darkMaskedImage=darkExposure.getMaskedImage(),
1914  expScale=expScale,
1915  darkScale=darkScale,
1916  invert=invert,
1917  trimToFit=self.config.doTrimToMatchCalib
1918  )
1919 
1920  def doLinearize(self, detector):
1921  """!Check if linearization is needed for the detector cameraGeom.
1922 
1923  Checks config.doLinearize and the linearity type of the first
1924  amplifier.
1925 
1926  Parameters
1927  ----------
1928  detector : `lsst.afw.cameraGeom.Detector`
1929  Detector to get linearity type from.
1930 
1931  Returns
1932  -------
1933  doLinearize : `Bool`
1934  If True, linearization should be performed.
1935  """
1936  return self.config.doLinearize and \
1937  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
1938 
1939  def flatCorrection(self, exposure, flatExposure, invert=False):
1940  """!Apply flat correction in place.
1941 
1942  Parameters
1943  ----------
1944  exposure : `lsst.afw.image.Exposure`
1945  Exposure to process.
1946  flatExposure : `lsst.afw.image.Exposure`
1947  Flat exposure of the same size as ``exposure``.
1948  invert : `Bool`, optional
1949  If True, unflatten an already flattened image.
1950 
1951  See Also
1952  --------
1953  lsst.ip.isr.isrFunctions.flatCorrection
1954  """
1955  isrFunctions.flatCorrection(
1956  maskedImage=exposure.getMaskedImage(),
1957  flatMaskedImage=flatExposure.getMaskedImage(),
1958  scalingType=self.config.flatScalingType,
1959  userScale=self.config.flatUserScale,
1960  invert=invert,
1961  trimToFit=self.config.doTrimToMatchCalib
1962  )
1963 
1964  def saturationDetection(self, exposure, amp):
1965  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1966 
1967  Parameters
1968  ----------
1969  exposure : `lsst.afw.image.Exposure`
1970  Exposure to process. Only the amplifier DataSec is processed.
1971  amp : `lsst.afw.table.AmpInfoCatalog`
1972  Amplifier detector data.
1973 
1974  See Also
1975  --------
1976  lsst.ip.isr.isrFunctions.makeThresholdMask
1977  """
1978  if not math.isnan(amp.getSaturation()):
1979  maskedImage = exposure.getMaskedImage()
1980  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1981  isrFunctions.makeThresholdMask(
1982  maskedImage=dataView,
1983  threshold=amp.getSaturation(),
1984  growFootprints=0,
1985  maskName=self.config.saturatedMaskName,
1986  )
1987 
1988  def saturationInterpolation(self, exposure):
1989  """!Interpolate over saturated pixels, in place.
1990 
1991  This method should be called after `saturationDetection`, to
1992  ensure that the saturated pixels have been identified in the
1993  SAT mask. It should also be called after `assembleCcd`, since
1994  saturated regions may cross amplifier boundaries.
1995 
1996  Parameters
1997  ----------
1998  exposure : `lsst.afw.image.Exposure`
1999  Exposure to process.
2000 
2001  See Also
2002  --------
2003  lsst.ip.isr.isrTask.saturationDetection
2004  lsst.ip.isr.isrFunctions.interpolateFromMask
2005  """
2006  isrFunctions.interpolateFromMask(
2007  maskedImage=exposure.getMaskedImage(),
2008  fwhm=self.config.fwhm,
2009  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2010  maskNameList=list(self.config.saturatedMaskName),
2011  )
2012 
2013  def suspectDetection(self, exposure, amp):
2014  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2015 
2016  Parameters
2017  ----------
2018  exposure : `lsst.afw.image.Exposure`
2019  Exposure to process. Only the amplifier DataSec is processed.
2020  amp : `lsst.afw.table.AmpInfoCatalog`
2021  Amplifier detector data.
2022 
2023  See Also
2024  --------
2025  lsst.ip.isr.isrFunctions.makeThresholdMask
2026 
2027  Notes
2028  -----
2029  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2030  This is intended to indicate pixels that may be affected by unknown systematics;
2031  for example if non-linearity corrections above a certain level are unstable
2032  then that would be a useful value for suspectLevel. A value of `nan` indicates
2033  that no such level exists and no pixels are to be masked as suspicious.
2034  """
2035  suspectLevel = amp.getSuspectLevel()
2036  if math.isnan(suspectLevel):
2037  return
2038 
2039  maskedImage = exposure.getMaskedImage()
2040  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2041  isrFunctions.makeThresholdMask(
2042  maskedImage=dataView,
2043  threshold=suspectLevel,
2044  growFootprints=0,
2045  maskName=self.config.suspectMaskName,
2046  )
2047 
2048  def maskDefect(self, exposure, defectBaseList):
2049  """!Mask defects using mask plane "BAD", in place.
2050 
2051  Parameters
2052  ----------
2053  exposure : `lsst.afw.image.Exposure`
2054  Exposure to process.
2055  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2056  `lsst.afw.image.DefectBase`.
2057  List of defects to mask.
2058 
2059  Notes
2060  -----
2061  Call this after CCD assembly, since defects may cross amplifier boundaries.
2062  """
2063  maskedImage = exposure.getMaskedImage()
2064  if not isinstance(defectBaseList, Defects):
2065  # Promotes DefectBase to Defect
2066  defectList = Defects(defectBaseList)
2067  else:
2068  defectList = defectBaseList
2069  defectList.maskPixels(maskedImage, maskName="BAD")
2070 
2071  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2072  """!Mask edge pixels with applicable mask plane.
2073 
2074  Parameters
2075  ----------
2076  exposure : `lsst.afw.image.Exposure`
2077  Exposure to process.
2078  numEdgePixels : `int`, optional
2079  Number of edge pixels to mask.
2080  maskPlane : `str`, optional
2081  Mask plane name to use.
2082  """
2083  maskedImage = exposure.getMaskedImage()
2084  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2085 
2086  if numEdgePixels > 0:
2087  goodBBox = maskedImage.getBBox()
2088  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2089  goodBBox.grow(-numEdgePixels)
2090  # Mask pixels outside goodBBox
2091  SourceDetectionTask.setEdgeBits(
2092  maskedImage,
2093  goodBBox,
2094  maskBitMask
2095  )
2096 
2097  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2098  """Mask and interpolate defects using mask plane "BAD", in place.
2099 
2100  Parameters
2101  ----------
2102  exposure : `lsst.afw.image.Exposure`
2103  Exposure to process.
2104  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2105  `lsst.afw.image.DefectBase`.
2106  List of defects to mask and interpolate.
2107 
2108  See Also
2109  --------
2110  lsst.ip.isr.isrTask.maskDefect()
2111  """
2112  self.maskDefect(exposure, defectBaseList)
2113  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2114  maskPlane="SUSPECT")
2115  isrFunctions.interpolateFromMask(
2116  maskedImage=exposure.getMaskedImage(),
2117  fwhm=self.config.fwhm,
2118  growSaturatedFootprints=0,
2119  maskNameList=["BAD"],
2120  )
2121 
2122  def maskNan(self, exposure):
2123  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2124 
2125  Parameters
2126  ----------
2127  exposure : `lsst.afw.image.Exposure`
2128  Exposure to process.
2129 
2130  Notes
2131  -----
2132  We mask over all NaNs, including those that are masked with
2133  other bits (because those may or may not be interpolated over
2134  later, and we want to remove all NaNs). Despite this
2135  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2136  the historical name.
2137  """
2138  maskedImage = exposure.getMaskedImage()
2139 
2140  # Find and mask NaNs
2141  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2142  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2143  numNans = maskNans(maskedImage, maskVal)
2144  self.metadata.set("NUMNANS", numNans)
2145  if numNans > 0:
2146  self.log.warn("There were %d unmasked NaNs.", numNans)
2147 
2148  def maskAndInterpolateNan(self, exposure):
2149  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2150 
2151  Parameters
2152  ----------
2153  exposure : `lsst.afw.image.Exposure`
2154  Exposure to process.
2155 
2156  See Also
2157  --------
2158  lsst.ip.isr.isrTask.maskNan()
2159  """
2160  self.maskNan(exposure)
2161  isrFunctions.interpolateFromMask(
2162  maskedImage=exposure.getMaskedImage(),
2163  fwhm=self.config.fwhm,
2164  growSaturatedFootprints=0,
2165  maskNameList=["UNMASKEDNAN"],
2166  )
2167 
2168  def measureBackground(self, exposure, IsrQaConfig=None):
2169  """Measure the image background in subgrids, for quality control purposes.
2170 
2171  Parameters
2172  ----------
2173  exposure : `lsst.afw.image.Exposure`
2174  Exposure to process.
2175  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2176  Configuration object containing parameters on which background
2177  statistics and subgrids to use.
2178  """
2179  if IsrQaConfig is not None:
2180  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2181  IsrQaConfig.flatness.nIter)
2182  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2183  statsControl.setAndMask(maskVal)
2184  maskedImage = exposure.getMaskedImage()
2185  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2186  skyLevel = stats.getValue(afwMath.MEDIAN)
2187  skySigma = stats.getValue(afwMath.STDEVCLIP)
2188  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2189  metadata = exposure.getMetadata()
2190  metadata.set('SKYLEVEL', skyLevel)
2191  metadata.set('SKYSIGMA', skySigma)
2192 
2193  # calcluating flatlevel over the subgrids
2194  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2195  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2196  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2197  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2198  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2199  skyLevels = numpy.zeros((nX, nY))
2200 
2201  for j in range(nY):
2202  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2203  for i in range(nX):
2204  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2205 
2206  xLLC = xc - meshXHalf
2207  yLLC = yc - meshYHalf
2208  xURC = xc + meshXHalf - 1
2209  yURC = yc + meshYHalf - 1
2210 
2211  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2212  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2213 
2214  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2215 
2216  good = numpy.where(numpy.isfinite(skyLevels))
2217  skyMedian = numpy.median(skyLevels[good])
2218  flatness = (skyLevels[good] - skyMedian) / skyMedian
2219  flatness_rms = numpy.std(flatness)
2220  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2221 
2222  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2223  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2224  nX, nY, flatness_pp, flatness_rms)
2225 
2226  metadata.set('FLATNESS_PP', float(flatness_pp))
2227  metadata.set('FLATNESS_RMS', float(flatness_rms))
2228  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2229  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2230  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2231 
2232  def roughZeroPoint(self, exposure):
2233  """Set an approximate magnitude zero point for the exposure.
2234 
2235  Parameters
2236  ----------
2237  exposure : `lsst.afw.image.Exposure`
2238  Exposure to process.
2239  """
2240  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2241  if filterName in self.config.fluxMag0T1:
2242  fluxMag0 = self.config.fluxMag0T1[filterName]
2243  else:
2244  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2245  fluxMag0 = self.config.defaultFluxMag0T1
2246 
2247  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2248  if not expTime > 0: # handle NaN as well as <= 0
2249  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2250  return
2251 
2252  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2253  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2254 
2255  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2256  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2257 
2258  Parameters
2259  ----------
2260  ccdExposure : `lsst.afw.image.Exposure`
2261  Exposure to process.
2262  fpPolygon : `lsst.afw.geom.Polygon`
2263  Polygon in focal plane coordinates.
2264  """
2265  # Get ccd corners in focal plane coordinates
2266  ccd = ccdExposure.getDetector()
2267  fpCorners = ccd.getCorners(FOCAL_PLANE)
2268  ccdPolygon = Polygon(fpCorners)
2269 
2270  # Get intersection of ccd corners with fpPolygon
2271  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2272 
2273  # Transform back to pixel positions and build new polygon
2274  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2275  validPolygon = Polygon(ccdPoints)
2276  ccdExposure.getInfo().setValidPolygon(validPolygon)
2277 
2278  @contextmanager
2279  def flatContext(self, exp, flat, dark=None):
2280  """Context manager that applies and removes flats and darks,
2281  if the task is configured to apply them.
2282 
2283  Parameters
2284  ----------
2285  exp : `lsst.afw.image.Exposure`
2286  Exposure to process.
2287  flat : `lsst.afw.image.Exposure`
2288  Flat exposure the same size as ``exp``.
2289  dark : `lsst.afw.image.Exposure`, optional
2290  Dark exposure the same size as ``exp``.
2291 
2292  Yields
2293  ------
2294  exp : `lsst.afw.image.Exposure`
2295  The flat and dark corrected exposure.
2296  """
2297  if self.config.doDark and dark is not None:
2298  self.darkCorrection(exp, dark)
2299  if self.config.doFlat:
2300  self.flatCorrection(exp, flat)
2301  try:
2302  yield exp
2303  finally:
2304  if self.config.doFlat:
2305  self.flatCorrection(exp, flat, invert=True)
2306  if self.config.doDark and dark is not None:
2307  self.darkCorrection(exp, dark, invert=True)
2308 
2309  def debugView(self, exposure, stepname):
2310  """Utility function to examine ISR exposure at different stages.
2311 
2312  Parameters
2313  ----------
2314  exposure : `lsst.afw.image.Exposure`
2315  Exposure to view.
2316  stepname : `str`
2317  State of processing to view.
2318  """
2319  frame = getDebugFrame(self._display, stepname)
2320  if frame:
2321  display = getDisplay(frame)
2322  display.scale('asinh', 'zscale')
2323  display.mtv(exposure)
2324  prompt = "Press Enter to continue [c]... "
2325  while True:
2326  ans = input(prompt).lower()
2327  if ans in ("", "c",):
2328  break
2329 
2330 
2331 class FakeAmp(object):
2332  """A Detector-like object that supports returning gain and saturation level
2333 
2334  This is used when the input exposure does not have a detector.
2335 
2336  Parameters
2337  ----------
2338  exposure : `lsst.afw.image.Exposure`
2339  Exposure to generate a fake amplifier for.
2340  config : `lsst.ip.isr.isrTaskConfig`
2341  Configuration to apply to the fake amplifier.
2342  """
2343 
2344  def __init__(self, exposure, config):
2345  self._bbox = exposure.getBBox(afwImage.LOCAL)
2347  self._gain = config.gain
2348  self._readNoise = config.readNoise
2349  self._saturation = config.saturation
2350 
2351  def getBBox(self):
2352  return self._bbox
2353 
2354  def getRawBBox(self):
2355  return self._bbox
2356 
2357  def getHasRawInfo(self):
2358  return True # but see getRawHorizontalOverscanBBox()
2359 
2361  return self._RawHorizontalOverscanBBox
2362 
2363  def getGain(self):
2364  return self._gain
2365 
2366  def getReadNoise(self):
2367  return self._readNoise
2368 
2369  def getSaturation(self):
2370  return self._saturation
2371 
2372  def getSuspectLevel(self):
2373  return float("NaN")
2374 
2375 
2376 class RunIsrConfig(pexConfig.Config):
2377  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2378 
2379 
2380 class RunIsrTask(pipeBase.CmdLineTask):
2381  """Task to wrap the default IsrTask to allow it to be retargeted.
2382 
2383  The standard IsrTask can be called directly from a command line
2384  program, but doing so removes the ability of the task to be
2385  retargeted. As most cameras override some set of the IsrTask
2386  methods, this would remove those data-specific methods in the
2387  output post-ISR images. This wrapping class fixes the issue,
2388  allowing identical post-ISR images to be generated by both the
2389  processCcd and isrTask code.
2390  """
2391  ConfigClass = RunIsrConfig
2392  _DefaultName = "runIsr"
2393 
2394  def __init__(self, *args, **kwargs):
2395  super().__init__(*args, **kwargs)
2396  self.makeSubtask("isr")
2397 
2398  def runDataRef(self, dataRef):
2399  """
2400  Parameters
2401  ----------
2402  dataRef : `lsst.daf.persistence.ButlerDataRef`
2403  data reference of the detector data to be processed
2404 
2405  Returns
2406  -------
2407  result : `pipeBase.Struct`
2408  Result struct with component:
2409 
2410  - exposure : `lsst.afw.image.Exposure`
2411  Post-ISR processed exposure.
2412  """
2413  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1440
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2168
def debugView(self, exposure, stepname)
Definition: isrTask.py:2309
def __init__(self, kwargs)
Definition: isrTask.py:795
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1532
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:850
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2071
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:804
def runDataRef(self, dataRef)
Definition: isrTask.py:2398
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:998
def __init__(self, args, kwargs)
Definition: isrTask.py:2394
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2232
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2097
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2360
def maskNan(self, exposure)
Definition: isrTask.py:2122
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2048
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1688
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1580
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1939
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1491
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1876
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1920
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2255
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1617
def __init__(self, config=None)
Definition: isrTask.py:173
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2279
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1826
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2148
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2013
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1988
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1964
def __init__(self, exposure, config)
Definition: isrTask.py:2344