lsst.ip.isr  19.0.0-7-gea0a0fe+4
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.PrerequisiteInput(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ImageF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ImageF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="MaskedImageF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  defects = cT.PrerequisiteInput(
112  name='defects',
113  doc="Input defect tables.",
114  storageClass="DefectsList",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  opticsTransmission = cT.PrerequisiteInput(
118  name="transmission_optics",
119  storageClass="TransmissionCurve",
120  doc="Transmission curve due to the optics.",
121  dimensions=["instrument", "calibration_label"],
122  )
123  filterTransmission = cT.PrerequisiteInput(
124  name="transmission_filter",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the filter.",
127  dimensions=["instrument", "physical_filter", "calibration_label"],
128  )
129  sensorTransmission = cT.PrerequisiteInput(
130  name="transmission_sensor",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the sensor.",
133  dimensions=["instrument", "calibration_label", "detector"],
134  )
135  atmosphereTransmission = cT.PrerequisiteInput(
136  name="transmission_atmosphere",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the atmosphere.",
139  dimensions=["instrument"],
140  )
141  illumMaskedImage = cT.PrerequisiteInput(
142  name="illum",
143  doc="Input illumination correction.",
144  storageClass="MaskedImageF",
145  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
146  )
147 
148  outputExposure = cT.Output(
149  name='postISRCCD',
150  doc="Output ISR processed exposure.",
151  storageClass="ExposureF",
152  dimensions=["instrument", "visit", "detector"],
153  )
154  preInterpExposure = cT.Output(
155  name='preInterpISRCCD',
156  doc="Output ISR processed exposure, with pixels left uninterpolated.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "visit", "detector"],
159  )
160  outputOssThumbnail = cT.Output(
161  name="OssThumb",
162  doc="Output Overscan-subtracted thumbnail image.",
163  storageClass="Thumbnail",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputFlattenedThumbnail = cT.Output(
167  name="FlattenedThumb",
168  doc="Output flat-corrected thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "visit", "detector"],
171  )
172 
173  def __init__(self, *, config=None):
174  super().__init__(config=config)
175 
176  if config.doBias is not True:
177  self.prerequisiteInputs.discard("bias")
178  if config.doLinearize is not True:
179  self.prerequisiteInputs.discard("linearizer")
180  if config.doCrosstalk is not True:
181  self.prerequisiteInputs.discard("crosstalkSources")
182  if config.doBrighterFatter is not True:
183  self.prerequisiteInputs.discard("bfKernel")
184  if config.doDefect is not True:
185  self.prerequisiteInputs.discard("defects")
186  if config.doDark is not True:
187  self.prerequisiteInputs.discard("dark")
188  if config.doFlat is not True:
189  self.prerequisiteInputs.discard("flat")
190  if config.doAttachTransmissionCurve is not True:
191  self.prerequisiteInputs.discard("opticsTransmission")
192  self.prerequisiteInputs.discard("filterTransmission")
193  self.prerequisiteInputs.discard("sensorTransmission")
194  self.prerequisiteInputs.discard("atmosphereTransmission")
195  if config.doUseOpticsTransmission is not True:
196  self.prerequisiteInputs.discard("opticsTransmission")
197  if config.doUseFilterTransmission is not True:
198  self.prerequisiteInputs.discard("filterTransmission")
199  if config.doUseSensorTransmission is not True:
200  self.prerequisiteInputs.discard("sensorTransmission")
201  if config.doUseAtmosphereTransmission is not True:
202  self.prerequisiteInputs.discard("atmosphereTransmission")
203  if config.doIlluminationCorrection is not True:
204  self.prerequisiteInputs.discard("illumMaskedImage")
205 
206  if config.doWrite is not True:
207  self.outputs.discard("outputExposure")
208  self.outputs.discard("preInterpExposure")
209  self.outputs.discard("outputFlattenedThumbnail")
210  self.outputs.discard("outputOssThumbnail")
211  if config.doSaveInterpPixels is not True:
212  self.outputs.discard("preInterpExposure")
213  if config.qa.doThumbnailOss is not True:
214  self.outputs.discard("outputOssThumbnail")
215  if config.qa.doThumbnailFlattened is not True:
216  self.outputs.discard("outputFlattenedThumbnail")
217 
218 
219 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
220  pipelineConnections=IsrTaskConnections):
221  """Configuration parameters for IsrTask.
222 
223  Items are grouped in the order in which they are executed by the task.
224  """
225  datasetType = pexConfig.Field(
226  dtype=str,
227  doc="Dataset type for input data; users will typically leave this alone, "
228  "but camera-specific ISR tasks will override it",
229  default="raw",
230  )
231 
232  fallbackFilterName = pexConfig.Field(
233  dtype=str,
234  doc="Fallback default filter name for calibrations.",
235  optional=True
236  )
237  useFallbackDate = pexConfig.Field(
238  dtype=bool,
239  doc="Pass observation date when using fallback filter.",
240  default=False,
241  )
242  expectWcs = pexConfig.Field(
243  dtype=bool,
244  default=True,
245  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
246  )
247  fwhm = pexConfig.Field(
248  dtype=float,
249  doc="FWHM of PSF in arcseconds.",
250  default=1.0,
251  )
252  qa = pexConfig.ConfigField(
253  dtype=isrQa.IsrQaConfig,
254  doc="QA related configuration options.",
255  )
256 
257  # Image conversion configuration
258  doConvertIntToFloat = pexConfig.Field(
259  dtype=bool,
260  doc="Convert integer raw images to floating point values?",
261  default=True,
262  )
263 
264  # Saturated pixel handling.
265  doSaturation = pexConfig.Field(
266  dtype=bool,
267  doc="Mask saturated pixels? NB: this is totally independent of the"
268  " interpolation option - this is ONLY setting the bits in the mask."
269  " To have them interpolated make sure doSaturationInterpolation=True",
270  default=True,
271  )
272  saturatedMaskName = pexConfig.Field(
273  dtype=str,
274  doc="Name of mask plane to use in saturation detection and interpolation",
275  default="SAT",
276  )
277  saturation = pexConfig.Field(
278  dtype=float,
279  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
280  default=float("NaN"),
281  )
282  growSaturationFootprintSize = pexConfig.Field(
283  dtype=int,
284  doc="Number of pixels by which to grow the saturation footprints",
285  default=1,
286  )
287 
288  # Suspect pixel handling.
289  doSuspect = pexConfig.Field(
290  dtype=bool,
291  doc="Mask suspect pixels?",
292  default=False,
293  )
294  suspectMaskName = pexConfig.Field(
295  dtype=str,
296  doc="Name of mask plane to use for suspect pixels",
297  default="SUSPECT",
298  )
299  numEdgeSuspect = pexConfig.Field(
300  dtype=int,
301  doc="Number of edge pixels to be flagged as untrustworthy.",
302  default=0,
303  )
304 
305  # Initial masking options.
306  doSetBadRegions = pexConfig.Field(
307  dtype=bool,
308  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
309  default=True,
310  )
311  badStatistic = pexConfig.ChoiceField(
312  dtype=str,
313  doc="How to estimate the average value for BAD regions.",
314  default='MEANCLIP',
315  allowed={
316  "MEANCLIP": "Correct using the (clipped) mean of good data",
317  "MEDIAN": "Correct using the median of the good data",
318  },
319  )
320 
321  # Overscan subtraction configuration.
322  doOverscan = pexConfig.Field(
323  dtype=bool,
324  doc="Do overscan subtraction?",
325  default=True,
326  )
327  overscanFitType = pexConfig.ChoiceField(
328  dtype=str,
329  doc="The method for fitting the overscan bias level.",
330  default='MEDIAN',
331  allowed={
332  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
333  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
334  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
335  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
336  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
337  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
338  "MEAN": "Correct using the mean of the overscan region",
339  "MEANCLIP": "Correct using a clipped mean of the overscan region",
340  "MEDIAN": "Correct using the median of the overscan region",
341  },
342  )
343  overscanOrder = pexConfig.Field(
344  dtype=int,
345  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
346  "or number of spline knots if overscan fit type is a spline."),
347  default=1,
348  )
349  overscanNumSigmaClip = pexConfig.Field(
350  dtype=float,
351  doc="Rejection threshold (sigma) for collapsing overscan before fit",
352  default=3.0,
353  )
354  overscanIsInt = pexConfig.Field(
355  dtype=bool,
356  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
357  default=True,
358  )
359  overscanNumLeadingColumnsToSkip = pexConfig.Field(
360  dtype=int,
361  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
362  default=0,
363  )
364  overscanNumTrailingColumnsToSkip = pexConfig.Field(
365  dtype=int,
366  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
367  default=0,
368  )
369  overscanMaxDev = pexConfig.Field(
370  dtype=float,
371  doc="Maximum deviation from the median for overscan",
372  default=1000.0, check=lambda x: x > 0
373  )
374  overscanBiasJump = pexConfig.Field(
375  dtype=bool,
376  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
377  default=False,
378  )
379  overscanBiasJumpKeyword = pexConfig.Field(
380  dtype=str,
381  doc="Header keyword containing information about devices.",
382  default="NO_SUCH_KEY",
383  )
384  overscanBiasJumpDevices = pexConfig.ListField(
385  dtype=str,
386  doc="List of devices that need piecewise overscan correction.",
387  default=(),
388  )
389  overscanBiasJumpLocation = pexConfig.Field(
390  dtype=int,
391  doc="Location of bias jump along y-axis.",
392  default=0,
393  )
394 
395  # Amplifier to CCD assembly configuration
396  doAssembleCcd = pexConfig.Field(
397  dtype=bool,
398  default=True,
399  doc="Assemble amp-level exposures into a ccd-level exposure?"
400  )
401  assembleCcd = pexConfig.ConfigurableField(
402  target=AssembleCcdTask,
403  doc="CCD assembly task",
404  )
405 
406  # General calibration configuration.
407  doAssembleIsrExposures = pexConfig.Field(
408  dtype=bool,
409  default=False,
410  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
411  )
412  doTrimToMatchCalib = pexConfig.Field(
413  dtype=bool,
414  default=False,
415  doc="Trim raw data to match calibration bounding boxes?"
416  )
417 
418  # Bias subtraction.
419  doBias = pexConfig.Field(
420  dtype=bool,
421  doc="Apply bias frame correction?",
422  default=True,
423  )
424  biasDataProductName = pexConfig.Field(
425  dtype=str,
426  doc="Name of the bias data product",
427  default="bias",
428  )
429 
430  # Variance construction
431  doVariance = pexConfig.Field(
432  dtype=bool,
433  doc="Calculate variance?",
434  default=True
435  )
436  gain = pexConfig.Field(
437  dtype=float,
438  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
439  default=float("NaN"),
440  )
441  readNoise = pexConfig.Field(
442  dtype=float,
443  doc="The read noise to use if no Detector is present in the Exposure",
444  default=0.0,
445  )
446  doEmpiricalReadNoise = pexConfig.Field(
447  dtype=bool,
448  default=False,
449  doc="Calculate empirical read noise instead of value from AmpInfo data?"
450  )
451 
452  # Linearization.
453  doLinearize = pexConfig.Field(
454  dtype=bool,
455  doc="Correct for nonlinearity of the detector's response?",
456  default=True,
457  )
458 
459  # Crosstalk.
460  doCrosstalk = pexConfig.Field(
461  dtype=bool,
462  doc="Apply intra-CCD crosstalk correction?",
463  default=False,
464  )
465  doCrosstalkBeforeAssemble = pexConfig.Field(
466  dtype=bool,
467  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
468  default=False,
469  )
470  crosstalk = pexConfig.ConfigurableField(
471  target=CrosstalkTask,
472  doc="Intra-CCD crosstalk correction",
473  )
474 
475  # Masking options.
476  doDefect = pexConfig.Field(
477  dtype=bool,
478  doc="Apply correction for CCD defects, e.g. hot pixels?",
479  default=True,
480  )
481  doNanMasking = pexConfig.Field(
482  dtype=bool,
483  doc="Mask NAN pixels?",
484  default=True,
485  )
486  doWidenSaturationTrails = pexConfig.Field(
487  dtype=bool,
488  doc="Widen bleed trails based on their width?",
489  default=True
490  )
491 
492  # Brighter-Fatter correction.
493  doBrighterFatter = pexConfig.Field(
494  dtype=bool,
495  default=False,
496  doc="Apply the brighter fatter correction"
497  )
498  brighterFatterLevel = pexConfig.ChoiceField(
499  dtype=str,
500  default="DETECTOR",
501  doc="The level at which to correct for brighter-fatter.",
502  allowed={
503  "AMP": "Every amplifier treated separately.",
504  "DETECTOR": "One kernel per detector",
505  }
506  )
507  brighterFatterMaxIter = pexConfig.Field(
508  dtype=int,
509  default=10,
510  doc="Maximum number of iterations for the brighter fatter correction"
511  )
512  brighterFatterThreshold = pexConfig.Field(
513  dtype=float,
514  default=1000,
515  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
516  " absolute value of the difference between the current corrected image and the one"
517  " from the previous iteration summed over all the pixels."
518  )
519  brighterFatterApplyGain = pexConfig.Field(
520  dtype=bool,
521  default=True,
522  doc="Should the gain be applied when applying the brighter fatter correction?"
523  )
524 
525  # Dark subtraction.
526  doDark = pexConfig.Field(
527  dtype=bool,
528  doc="Apply dark frame correction?",
529  default=True,
530  )
531  darkDataProductName = pexConfig.Field(
532  dtype=str,
533  doc="Name of the dark data product",
534  default="dark",
535  )
536 
537  # Camera-specific stray light removal.
538  doStrayLight = pexConfig.Field(
539  dtype=bool,
540  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
541  default=False,
542  )
543  strayLight = pexConfig.ConfigurableField(
544  target=StrayLightTask,
545  doc="y-band stray light correction"
546  )
547 
548  # Flat correction.
549  doFlat = pexConfig.Field(
550  dtype=bool,
551  doc="Apply flat field correction?",
552  default=True,
553  )
554  flatDataProductName = pexConfig.Field(
555  dtype=str,
556  doc="Name of the flat data product",
557  default="flat",
558  )
559  flatScalingType = pexConfig.ChoiceField(
560  dtype=str,
561  doc="The method for scaling the flat on the fly.",
562  default='USER',
563  allowed={
564  "USER": "Scale by flatUserScale",
565  "MEAN": "Scale by the inverse of the mean",
566  "MEDIAN": "Scale by the inverse of the median",
567  },
568  )
569  flatUserScale = pexConfig.Field(
570  dtype=float,
571  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
572  default=1.0,
573  )
574  doTweakFlat = pexConfig.Field(
575  dtype=bool,
576  doc="Tweak flats to match observed amplifier ratios?",
577  default=False
578  )
579 
580  # Amplifier normalization based on gains instead of using flats configuration.
581  doApplyGains = pexConfig.Field(
582  dtype=bool,
583  doc="Correct the amplifiers for their gains instead of applying flat correction",
584  default=False,
585  )
586  normalizeGains = pexConfig.Field(
587  dtype=bool,
588  doc="Normalize all the amplifiers in each CCD to have the same median value.",
589  default=False,
590  )
591 
592  # Fringe correction.
593  doFringe = pexConfig.Field(
594  dtype=bool,
595  doc="Apply fringe correction?",
596  default=True,
597  )
598  fringe = pexConfig.ConfigurableField(
599  target=FringeTask,
600  doc="Fringe subtraction task",
601  )
602  fringeAfterFlat = pexConfig.Field(
603  dtype=bool,
604  doc="Do fringe subtraction after flat-fielding?",
605  default=True,
606  )
607 
608  # Distortion model application.
609  doAddDistortionModel = pexConfig.Field(
610  dtype=bool,
611  doc="Apply a distortion model based on camera geometry to the WCS?",
612  default=True,
613  deprecated=("Camera geometry is incorporated when reading the raw files."
614  " This option no longer is used, and will be removed after v19.")
615  )
616 
617  # Initial CCD-level background statistics options.
618  doMeasureBackground = pexConfig.Field(
619  dtype=bool,
620  doc="Measure the background level on the reduced image?",
621  default=False,
622  )
623 
624  # Camera-specific masking configuration.
625  doCameraSpecificMasking = pexConfig.Field(
626  dtype=bool,
627  doc="Mask camera-specific bad regions?",
628  default=False,
629  )
630  masking = pexConfig.ConfigurableField(
631  target=MaskingTask,
632  doc="Masking task."
633  )
634 
635  # Interpolation options.
636 
637  doInterpolate = pexConfig.Field(
638  dtype=bool,
639  doc="Interpolate masked pixels?",
640  default=True,
641  )
642  doSaturationInterpolation = pexConfig.Field(
643  dtype=bool,
644  doc="Perform interpolation over pixels masked as saturated?"
645  " NB: This is independent of doSaturation; if that is False this plane"
646  " will likely be blank, resulting in a no-op here.",
647  default=True,
648  )
649  doNanInterpolation = pexConfig.Field(
650  dtype=bool,
651  doc="Perform interpolation over pixels masked as NaN?"
652  " NB: This is independent of doNanMasking; if that is False this plane"
653  " will likely be blank, resulting in a no-op here.",
654  default=True,
655  )
656  doNanInterpAfterFlat = pexConfig.Field(
657  dtype=bool,
658  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
659  "also have to interpolate them before flat-fielding."),
660  default=False,
661  )
662  maskListToInterpolate = pexConfig.ListField(
663  dtype=str,
664  doc="List of mask planes that should be interpolated.",
665  default=['SAT', 'BAD', 'UNMASKEDNAN'],
666  )
667  doSaveInterpPixels = pexConfig.Field(
668  dtype=bool,
669  doc="Save a copy of the pre-interpolated pixel values?",
670  default=False,
671  )
672 
673  # Default photometric calibration options.
674  fluxMag0T1 = pexConfig.DictField(
675  keytype=str,
676  itemtype=float,
677  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
678  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
679  ))
680  )
681  defaultFluxMag0T1 = pexConfig.Field(
682  dtype=float,
683  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
684  default=pow(10.0, 0.4*28.0)
685  )
686 
687  # Vignette correction configuration.
688  doVignette = pexConfig.Field(
689  dtype=bool,
690  doc="Apply vignetting parameters?",
691  default=False,
692  )
693  vignette = pexConfig.ConfigurableField(
694  target=VignetteTask,
695  doc="Vignetting task.",
696  )
697 
698  # Transmission curve configuration.
699  doAttachTransmissionCurve = pexConfig.Field(
700  dtype=bool,
701  default=False,
702  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
703  )
704  doUseOpticsTransmission = pexConfig.Field(
705  dtype=bool,
706  default=True,
707  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
708  )
709  doUseFilterTransmission = pexConfig.Field(
710  dtype=bool,
711  default=True,
712  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
713  )
714  doUseSensorTransmission = pexConfig.Field(
715  dtype=bool,
716  default=True,
717  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
718  )
719  doUseAtmosphereTransmission = pexConfig.Field(
720  dtype=bool,
721  default=True,
722  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
723  )
724 
725  # Illumination correction.
726  doIlluminationCorrection = pexConfig.Field(
727  dtype=bool,
728  default=False,
729  doc="Perform illumination correction?"
730  )
731  illuminationCorrectionDataProductName = pexConfig.Field(
732  dtype=str,
733  doc="Name of the illumination correction data product.",
734  default="illumcor",
735  )
736  illumScale = pexConfig.Field(
737  dtype=float,
738  doc="Scale factor for the illumination correction.",
739  default=1.0,
740  )
741  illumFilters = pexConfig.ListField(
742  dtype=str,
743  default=[],
744  doc="Only perform illumination correction for these filters."
745  )
746 
747  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
748  doWrite = pexConfig.Field(
749  dtype=bool,
750  doc="Persist postISRCCD?",
751  default=True,
752  )
753 
754  def validate(self):
755  super().validate()
756  if self.doFlat and self.doApplyGains:
757  raise ValueError("You may not specify both doFlat and doApplyGains")
758  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
759  self.config.maskListToInterpolate.append("SAT")
760  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
761  self.config.maskListToInterpolate.append("UNMASKEDNAN")
762 
763 
764 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
765  """Apply common instrument signature correction algorithms to a raw frame.
766 
767  The process for correcting imaging data is very similar from
768  camera to camera. This task provides a vanilla implementation of
769  doing these corrections, including the ability to turn certain
770  corrections off if they are not needed. The inputs to the primary
771  method, `run()`, are a raw exposure to be corrected and the
772  calibration data products. The raw input is a single chip sized
773  mosaic of all amps including overscans and other non-science
774  pixels. The method `runDataRef()` identifies and defines the
775  calibration data products, and is intended for use by a
776  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
777  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
778  subclassed for different camera, although the most camera specific
779  methods have been split into subtasks that can be redirected
780  appropriately.
781 
782  The __init__ method sets up the subtasks for ISR processing, using
783  the defaults from `lsst.ip.isr`.
784 
785  Parameters
786  ----------
787  args : `list`
788  Positional arguments passed to the Task constructor. None used at this time.
789  kwargs : `dict`, optional
790  Keyword arguments passed on to the Task constructor. None used at this time.
791  """
792  ConfigClass = IsrTaskConfig
793  _DefaultName = "isr"
794 
795  def __init__(self, **kwargs):
796  super().__init__(**kwargs)
797  self.makeSubtask("assembleCcd")
798  self.makeSubtask("crosstalk")
799  self.makeSubtask("strayLight")
800  self.makeSubtask("fringe")
801  self.makeSubtask("masking")
802  self.makeSubtask("vignette")
803 
804  def runQuantum(self, butlerQC, inputRefs, outputRefs):
805  inputs = butlerQC.get(inputRefs)
806 
807  try:
808  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
809  except Exception as e:
810  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
811  (inputRefs, e))
812 
813  inputs['isGen3'] = True
814 
815  detector = inputs['ccdExposure'].getDetector()
816  if self.doLinearize(detector) is True:
817  if 'linearizer' not in inputs:
818  linearityName = detector.getAmplifiers()[0].getLinearityType()
819  inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
820 
821  if self.config.doDefect is True:
822  if "defects" in inputs and inputs['defects'] is not None:
823  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
824  # masking expects a list of defects defined by their bounding box
825  if not isinstance(inputs["defects"], Defects):
826  inputs["defects"] = Defects.fromTable(inputs["defects"])
827 
828  # TODO: DM-22776 add retrieval for brighter-fatter kernel for Gen3
829  # if we can get HSC to use a new kernel, or just translate the old
830  # one to the new format we could drop the whole new/old style support
831 
832  # Broken: DM-17169
833  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
834  # fixed for non-HSC cameras in the future.
835  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
836  # if self.config.doCrosstalk else None)
837 
838  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
839  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
840  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
841  expId=expId,
842  assembler=self.assembleCcd
843  if self.config.doAssembleIsrExposures else None)
844  else:
845  inputs['fringes'] = pipeBase.Struct(fringes=None)
846 
847  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
848  if 'strayLightData' not in inputs:
849  inputs['strayLightData'] = None
850 
851  outputs = self.run(**inputs)
852  butlerQC.put(outputs, outputRefs)
853 
854  def readIsrData(self, dataRef, rawExposure):
855  """!Retrieve necessary frames for instrument signature removal.
856 
857  Pre-fetching all required ISR data products limits the IO
858  required by the ISR. Any conflict between the calibration data
859  available and that needed for ISR is also detected prior to
860  doing processing, allowing it to fail quickly.
861 
862  Parameters
863  ----------
864  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
865  Butler reference of the detector data to be processed
866  rawExposure : `afw.image.Exposure`
867  The raw exposure that will later be corrected with the
868  retrieved calibration data; should not be modified in this
869  method.
870 
871  Returns
872  -------
873  result : `lsst.pipe.base.Struct`
874  Result struct with components (which may be `None`):
875  - ``bias``: bias calibration frame (`afw.image.Exposure`)
876  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
877  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
878  - ``dark``: dark calibration frame (`afw.image.Exposure`)
879  - ``flat``: flat calibration frame (`afw.image.Exposure`)
880  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
881  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
882  - ``fringes``: `lsst.pipe.base.Struct` with components:
883  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
884  - ``seed``: random seed derived from the ccdExposureId for random
885  number generator (`uint32`).
886  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
887  A ``TransmissionCurve`` that represents the throughput of the optics,
888  to be evaluated in focal-plane coordinates.
889  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
890  A ``TransmissionCurve`` that represents the throughput of the filter
891  itself, to be evaluated in focal-plane coordinates.
892  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
893  A ``TransmissionCurve`` that represents the throughput of the sensor
894  itself, to be evaluated in post-assembly trimmed detector coordinates.
895  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
896  A ``TransmissionCurve`` that represents the throughput of the
897  atmosphere, assumed to be spatially constant.
898  - ``strayLightData`` : `object`
899  An opaque object containing calibration information for
900  stray-light correction. If `None`, no correction will be
901  performed.
902  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
903 
904  Raises
905  ------
906  NotImplementedError :
907  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
908  """
909  try:
910  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
911  dateObs = dateObs.toPython().isoformat()
912  except RuntimeError:
913  self.log.warn("Unable to identify dateObs for rawExposure.")
914  dateObs = None
915 
916  ccd = rawExposure.getDetector()
917  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
918  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
919  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
920  if self.config.doBias else None)
921  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
922  linearizer = (dataRef.get("linearizer", immediate=True)
923  if self.doLinearize(ccd) else None)
924  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
925  if self.config.doCrosstalk else None)
926  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
927  if self.config.doDark else None)
928  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
929  dateObs=dateObs)
930  if self.config.doFlat else None)
931 
932  brighterFatterKernel = None
933  brighterFatterGains = None
934  if self.config.doBrighterFatter is True:
935  try:
936  # Use the new-style cp_pipe version of the kernel if it exists
937  # If using a new-style kernel, always use the self-consistent
938  # gains, i.e. the ones inside the kernel object itself
939  brighterFatterKernel = dataRef.get("brighterFatterKernel")
940  brighterFatterGains = brighterFatterKernel.gain
941  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
942  except NoResults:
943  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
944  brighterFatterKernel = dataRef.get("bfKernel")
945  self.log.info("Old style bright-fatter kernel (np.array) loaded")
946  except NoResults:
947  brighterFatterKernel = None
948  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
949  # If the kernel is not an ndarray, it's the cp_pipe version
950  # so extract the kernel for this detector, or raise an error
951  if self.config.brighterFatterLevel == 'DETECTOR':
952  if brighterFatterKernel.detectorKernel:
953  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
954  elif brighterFatterKernel.detectorKernelFromAmpKernels:
955  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
956  else:
957  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
958  else:
959  # TODO DM-15631 for implementing this
960  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
961 
962  defectList = (dataRef.get("defects")
963  if self.config.doDefect else None)
964  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
965  if self.config.doAssembleIsrExposures else None)
966  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
967  else pipeBase.Struct(fringes=None))
968 
969  if self.config.doAttachTransmissionCurve:
970  opticsTransmission = (dataRef.get("transmission_optics")
971  if self.config.doUseOpticsTransmission else None)
972  filterTransmission = (dataRef.get("transmission_filter")
973  if self.config.doUseFilterTransmission else None)
974  sensorTransmission = (dataRef.get("transmission_sensor")
975  if self.config.doUseSensorTransmission else None)
976  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
977  if self.config.doUseAtmosphereTransmission else None)
978  else:
979  opticsTransmission = None
980  filterTransmission = None
981  sensorTransmission = None
982  atmosphereTransmission = None
983 
984  if self.config.doStrayLight:
985  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
986  else:
987  strayLightData = None
988 
989  illumMaskedImage = (self.getIsrExposure(dataRef,
990  self.config.illuminationCorrectionDataProductName).getMaskedImage()
991  if (self.config.doIlluminationCorrection and
992  filterName in self.config.illumFilters)
993  else None)
994 
995  # Struct should include only kwargs to run()
996  return pipeBase.Struct(bias=biasExposure,
997  linearizer=linearizer,
998  crosstalkSources=crosstalkSources,
999  dark=darkExposure,
1000  flat=flatExposure,
1001  bfKernel=brighterFatterKernel,
1002  bfGains=brighterFatterGains,
1003  defects=defectList,
1004  fringes=fringeStruct,
1005  opticsTransmission=opticsTransmission,
1006  filterTransmission=filterTransmission,
1007  sensorTransmission=sensorTransmission,
1008  atmosphereTransmission=atmosphereTransmission,
1009  strayLightData=strayLightData,
1010  illumMaskedImage=illumMaskedImage
1011  )
1012 
1013  @pipeBase.timeMethod
1014  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1015  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1016  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1017  sensorTransmission=None, atmosphereTransmission=None,
1018  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1019  isGen3=False,
1020  ):
1021  """!Perform instrument signature removal on an exposure.
1022 
1023  Steps included in the ISR processing, in order performed, are:
1024  - saturation and suspect pixel masking
1025  - overscan subtraction
1026  - CCD assembly of individual amplifiers
1027  - bias subtraction
1028  - variance image construction
1029  - linearization of non-linear response
1030  - crosstalk masking
1031  - brighter-fatter correction
1032  - dark subtraction
1033  - fringe correction
1034  - stray light subtraction
1035  - flat correction
1036  - masking of known defects and camera specific features
1037  - vignette calculation
1038  - appending transmission curve and distortion model
1039 
1040  Parameters
1041  ----------
1042  ccdExposure : `lsst.afw.image.Exposure`
1043  The raw exposure that is to be run through ISR. The
1044  exposure is modified by this method.
1045  camera : `lsst.afw.cameraGeom.Camera`, optional
1046  The camera geometry for this exposure. Used to select the
1047  distortion model appropriate for this data.
1048  bias : `lsst.afw.image.Exposure`, optional
1049  Bias calibration frame.
1050  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1051  Functor for linearization.
1052  crosstalkSources : `list`, optional
1053  List of possible crosstalk sources.
1054  dark : `lsst.afw.image.Exposure`, optional
1055  Dark calibration frame.
1056  flat : `lsst.afw.image.Exposure`, optional
1057  Flat calibration frame.
1058  bfKernel : `numpy.ndarray`, optional
1059  Brighter-fatter kernel.
1060  bfGains : `dict` of `float`, optional
1061  Gains used to override the detector's nominal gains for the
1062  brighter-fatter correction. A dict keyed by amplifier name for
1063  the detector in question.
1064  defects : `lsst.meas.algorithms.Defects`, optional
1065  List of defects.
1066  fringes : `lsst.pipe.base.Struct`, optional
1067  Struct containing the fringe correction data, with
1068  elements:
1069  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1070  - ``seed``: random seed derived from the ccdExposureId for random
1071  number generator (`uint32`)
1072  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1073  A ``TransmissionCurve`` that represents the throughput of the optics,
1074  to be evaluated in focal-plane coordinates.
1075  filterTransmission : `lsst.afw.image.TransmissionCurve`
1076  A ``TransmissionCurve`` that represents the throughput of the filter
1077  itself, to be evaluated in focal-plane coordinates.
1078  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1079  A ``TransmissionCurve`` that represents the throughput of the sensor
1080  itself, to be evaluated in post-assembly trimmed detector coordinates.
1081  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1082  A ``TransmissionCurve`` that represents the throughput of the
1083  atmosphere, assumed to be spatially constant.
1084  detectorNum : `int`, optional
1085  The integer number for the detector to process.
1086  isGen3 : bool, optional
1087  Flag this call to run() as using the Gen3 butler environment.
1088  strayLightData : `object`, optional
1089  Opaque object containing calibration information for stray-light
1090  correction. If `None`, no correction will be performed.
1091  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1092  Illumination correction image.
1093 
1094  Returns
1095  -------
1096  result : `lsst.pipe.base.Struct`
1097  Result struct with component:
1098  - ``exposure`` : `afw.image.Exposure`
1099  The fully ISR corrected exposure.
1100  - ``outputExposure`` : `afw.image.Exposure`
1101  An alias for `exposure`
1102  - ``ossThumb`` : `numpy.ndarray`
1103  Thumbnail image of the exposure after overscan subtraction.
1104  - ``flattenedThumb`` : `numpy.ndarray`
1105  Thumbnail image of the exposure after flat-field correction.
1106 
1107  Raises
1108  ------
1109  RuntimeError
1110  Raised if a configuration option is set to True, but the
1111  required calibration data has not been specified.
1112 
1113  Notes
1114  -----
1115  The current processed exposure can be viewed by setting the
1116  appropriate lsstDebug entries in the `debug.display`
1117  dictionary. The names of these entries correspond to some of
1118  the IsrTaskConfig Boolean options, with the value denoting the
1119  frame to use. The exposure is shown inside the matching
1120  option check and after the processing of that step has
1121  finished. The steps with debug points are:
1122 
1123  doAssembleCcd
1124  doBias
1125  doCrosstalk
1126  doBrighterFatter
1127  doDark
1128  doFringe
1129  doStrayLight
1130  doFlat
1131 
1132  In addition, setting the "postISRCCD" entry displays the
1133  exposure after all ISR processing has finished.
1134 
1135  """
1136 
1137  if isGen3 is True:
1138  # Gen3 currently cannot automatically do configuration overrides.
1139  # DM-15257 looks to discuss this issue.
1140  # Configure input exposures;
1141  if detectorNum is None:
1142  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1143 
1144  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1145  bias = self.ensureExposure(bias, camera, detectorNum)
1146  dark = self.ensureExposure(dark, camera, detectorNum)
1147  flat = self.ensureExposure(flat, camera, detectorNum)
1148  else:
1149  if isinstance(ccdExposure, ButlerDataRef):
1150  return self.runDataRef(ccdExposure)
1151 
1152  ccd = ccdExposure.getDetector()
1153  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1154 
1155  if not ccd:
1156  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1157  ccd = [FakeAmp(ccdExposure, self.config)]
1158 
1159  # Validate Input
1160  if self.config.doBias and bias is None:
1161  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1162  if self.doLinearize(ccd) and linearizer is None:
1163  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1164  if self.config.doBrighterFatter and bfKernel is None:
1165  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1166  if self.config.doDark and dark is None:
1167  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1168  if self.config.doFlat and flat is None:
1169  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1170  if self.config.doDefect and defects is None:
1171  raise RuntimeError("Must supply defects if config.doDefect=True.")
1172  if (self.config.doFringe and filterName in self.fringe.config.filters and
1173  fringes.fringes is None):
1174  # The `fringes` object needs to be a pipeBase.Struct, as
1175  # we use it as a `dict` for the parameters of
1176  # `FringeTask.run()`. The `fringes.fringes` `list` may
1177  # not be `None` if `doFringe=True`. Otherwise, raise.
1178  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1179  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1180  illumMaskedImage is None):
1181  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1182 
1183  # Begin ISR processing.
1184  if self.config.doConvertIntToFloat:
1185  self.log.info("Converting exposure to floating point values.")
1186  ccdExposure = self.convertIntToFloat(ccdExposure)
1187 
1188  # Amplifier level processing.
1189  overscans = []
1190  for amp in ccd:
1191  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1192  if ccdExposure.getBBox().contains(amp.getBBox()):
1193  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1194  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1195 
1196  if self.config.doOverscan and not badAmp:
1197  # Overscan correction on amp-by-amp basis.
1198  overscanResults = self.overscanCorrection(ccdExposure, amp)
1199  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1200  if overscanResults is not None and \
1201  self.config.qa is not None and self.config.qa.saveStats is True:
1202  if isinstance(overscanResults.overscanFit, float):
1203  qaMedian = overscanResults.overscanFit
1204  qaStdev = float("NaN")
1205  else:
1206  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1207  afwMath.MEDIAN | afwMath.STDEVCLIP)
1208  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1209  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1210 
1211  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1212  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1213  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1214  amp.getName(), qaMedian, qaStdev)
1215  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1216  else:
1217  if badAmp:
1218  self.log.warn("Amplifier %s is bad.", amp.getName())
1219  overscanResults = None
1220 
1221  overscans.append(overscanResults if overscanResults is not None else None)
1222  else:
1223  self.log.info("Skipped OSCAN for %s.", amp.getName())
1224 
1225  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1226  self.log.info("Applying crosstalk correction.")
1227  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1228  self.debugView(ccdExposure, "doCrosstalk")
1229 
1230  if self.config.doAssembleCcd:
1231  self.log.info("Assembling CCD from amplifiers.")
1232  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1233 
1234  if self.config.expectWcs and not ccdExposure.getWcs():
1235  self.log.warn("No WCS found in input exposure.")
1236  self.debugView(ccdExposure, "doAssembleCcd")
1237 
1238  ossThumb = None
1239  if self.config.qa.doThumbnailOss:
1240  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1241 
1242  if self.config.doBias:
1243  self.log.info("Applying bias correction.")
1244  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1245  trimToFit=self.config.doTrimToMatchCalib)
1246  self.debugView(ccdExposure, "doBias")
1247 
1248  if self.config.doVariance:
1249  for amp, overscanResults in zip(ccd, overscans):
1250  if ccdExposure.getBBox().contains(amp.getBBox()):
1251  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1252  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1253  if overscanResults is not None:
1254  self.updateVariance(ampExposure, amp,
1255  overscanImage=overscanResults.overscanImage)
1256  else:
1257  self.updateVariance(ampExposure, amp,
1258  overscanImage=None)
1259  if self.config.qa is not None and self.config.qa.saveStats is True:
1260  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1261  afwMath.MEDIAN | afwMath.STDEVCLIP)
1262  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1263  qaStats.getValue(afwMath.MEDIAN))
1264  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1265  qaStats.getValue(afwMath.STDEVCLIP))
1266  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1267  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1268  qaStats.getValue(afwMath.STDEVCLIP))
1269 
1270  if self.doLinearize(ccd):
1271  self.log.info("Applying linearizer.")
1272  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1273 
1274  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1275  self.log.info("Applying crosstalk correction.")
1276  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1277  self.debugView(ccdExposure, "doCrosstalk")
1278 
1279  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1280  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1281  if self.config.doDefect:
1282  self.log.info("Masking defects.")
1283  self.maskDefect(ccdExposure, defects)
1284 
1285  if self.config.numEdgeSuspect > 0:
1286  self.log.info("Masking edges as SUSPECT.")
1287  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1288  maskPlane="SUSPECT")
1289 
1290  if self.config.doNanMasking:
1291  self.log.info("Masking NAN value pixels.")
1292  self.maskNan(ccdExposure)
1293 
1294  if self.config.doWidenSaturationTrails:
1295  self.log.info("Widening saturation trails.")
1296  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1297 
1298  if self.config.doCameraSpecificMasking:
1299  self.log.info("Masking regions for camera specific reasons.")
1300  self.masking.run(ccdExposure)
1301 
1302  if self.config.doBrighterFatter:
1303  # We need to apply flats and darks before we can interpolate, and we
1304  # need to interpolate before we do B-F, but we do B-F without the
1305  # flats and darks applied so we can work in units of electrons or holes.
1306  # This context manager applies and then removes the darks and flats.
1307  #
1308  # We also do not want to interpolate values here, so operate on temporary
1309  # images so we can apply only the BF-correction and roll back the
1310  # interpolation.
1311  interpExp = ccdExposure.clone()
1312  with self.flatContext(interpExp, flat, dark):
1313  isrFunctions.interpolateFromMask(
1314  maskedImage=interpExp.getMaskedImage(),
1315  fwhm=self.config.fwhm,
1316  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1317  maskNameList=self.config.maskListToInterpolate
1318  )
1319  bfExp = interpExp.clone()
1320 
1321  self.log.info("Applying brighter fatter correction.")
1322  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1323  self.config.brighterFatterMaxIter,
1324  self.config.brighterFatterThreshold,
1325  self.config.brighterFatterApplyGain,
1326  bfGains)
1327  if bfResults[1] == self.config.brighterFatterMaxIter:
1328  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1329  bfResults[0])
1330  else:
1331  self.log.info("Finished brighter fatter correction in %d iterations.",
1332  bfResults[1])
1333  image = ccdExposure.getMaskedImage().getImage()
1334  bfCorr = bfExp.getMaskedImage().getImage()
1335  bfCorr -= interpExp.getMaskedImage().getImage()
1336  image += bfCorr
1337 
1338  # Applying the brighter-fatter correction applies a
1339  # convolution to the science image. At the edges this
1340  # convolution may not have sufficient valid pixels to
1341  # produce a valid correction. Mark pixels within the size
1342  # of the brighter-fatter kernel as EDGE to warn of this
1343  # fact.
1344  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1345  maskPlane="EDGE")
1346  self.log.warn("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1347 
1348  self.debugView(ccdExposure, "doBrighterFatter")
1349 
1350  if self.config.doDark:
1351  self.log.info("Applying dark correction.")
1352  self.darkCorrection(ccdExposure, dark)
1353  self.debugView(ccdExposure, "doDark")
1354 
1355  if self.config.doFringe and not self.config.fringeAfterFlat:
1356  self.log.info("Applying fringe correction before flat.")
1357  self.fringe.run(ccdExposure, **fringes.getDict())
1358  self.debugView(ccdExposure, "doFringe")
1359 
1360  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1361  self.log.info("Checking strayLight correction.")
1362  self.strayLight.run(ccdExposure, strayLightData)
1363  self.debugView(ccdExposure, "doStrayLight")
1364 
1365  if self.config.doFlat:
1366  self.log.info("Applying flat correction.")
1367  self.flatCorrection(ccdExposure, flat)
1368  self.debugView(ccdExposure, "doFlat")
1369 
1370  if self.config.doApplyGains:
1371  self.log.info("Applying gain correction instead of flat.")
1372  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1373 
1374  if self.config.doFringe and self.config.fringeAfterFlat:
1375  self.log.info("Applying fringe correction after flat.")
1376  self.fringe.run(ccdExposure, **fringes.getDict())
1377 
1378  if self.config.doVignette:
1379  self.log.info("Constructing Vignette polygon.")
1380  self.vignettePolygon = self.vignette.run(ccdExposure)
1381 
1382  if self.config.vignette.doWriteVignettePolygon:
1383  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1384 
1385  if self.config.doAttachTransmissionCurve:
1386  self.log.info("Adding transmission curves.")
1387  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1388  filterTransmission=filterTransmission,
1389  sensorTransmission=sensorTransmission,
1390  atmosphereTransmission=atmosphereTransmission)
1391 
1392  flattenedThumb = None
1393  if self.config.qa.doThumbnailFlattened:
1394  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1395 
1396  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1397  self.log.info("Performing illumination correction.")
1398  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1399  illumMaskedImage, illumScale=self.config.illumScale,
1400  trimToFit=self.config.doTrimToMatchCalib)
1401 
1402  preInterpExp = None
1403  if self.config.doSaveInterpPixels:
1404  preInterpExp = ccdExposure.clone()
1405 
1406  # Reset and interpolate bad pixels.
1407  #
1408  # Large contiguous bad regions (which should have the BAD mask
1409  # bit set) should have their values set to the image median.
1410  # This group should include defects and bad amplifiers. As the
1411  # area covered by these defects are large, there's little
1412  # reason to expect that interpolation would provide a more
1413  # useful value.
1414  #
1415  # Smaller defects can be safely interpolated after the larger
1416  # regions have had their pixel values reset. This ensures
1417  # that the remaining defects adjacent to bad amplifiers (as an
1418  # example) do not attempt to interpolate extreme values.
1419  if self.config.doSetBadRegions:
1420  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1421  if badPixelCount > 0:
1422  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1423 
1424  if self.config.doInterpolate:
1425  self.log.info("Interpolating masked pixels.")
1426  isrFunctions.interpolateFromMask(
1427  maskedImage=ccdExposure.getMaskedImage(),
1428  fwhm=self.config.fwhm,
1429  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1430  maskNameList=list(self.config.maskListToInterpolate)
1431  )
1432 
1433  self.roughZeroPoint(ccdExposure)
1434 
1435  if self.config.doMeasureBackground:
1436  self.log.info("Measuring background level.")
1437  self.measureBackground(ccdExposure, self.config.qa)
1438 
1439  if self.config.qa is not None and self.config.qa.saveStats is True:
1440  for amp in ccd:
1441  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1442  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1443  afwMath.MEDIAN | afwMath.STDEVCLIP)
1444  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1445  qaStats.getValue(afwMath.MEDIAN))
1446  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1447  qaStats.getValue(afwMath.STDEVCLIP))
1448  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1449  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1450  qaStats.getValue(afwMath.STDEVCLIP))
1451 
1452  self.debugView(ccdExposure, "postISRCCD")
1453 
1454  return pipeBase.Struct(
1455  exposure=ccdExposure,
1456  ossThumb=ossThumb,
1457  flattenedThumb=flattenedThumb,
1458 
1459  preInterpolatedExposure=preInterpExp,
1460  outputExposure=ccdExposure,
1461  outputOssThumbnail=ossThumb,
1462  outputFlattenedThumbnail=flattenedThumb,
1463  )
1464 
1465  @pipeBase.timeMethod
1466  def runDataRef(self, sensorRef):
1467  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1468 
1469  This method contains the `CmdLineTask` interface to the ISR
1470  processing. All IO is handled here, freeing the `run()` method
1471  to manage only pixel-level calculations. The steps performed
1472  are:
1473  - Read in necessary detrending/isr/calibration data.
1474  - Process raw exposure in `run()`.
1475  - Persist the ISR-corrected exposure as "postISRCCD" if
1476  config.doWrite=True.
1477 
1478  Parameters
1479  ----------
1480  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1481  DataRef of the detector data to be processed
1482 
1483  Returns
1484  -------
1485  result : `lsst.pipe.base.Struct`
1486  Result struct with component:
1487  - ``exposure`` : `afw.image.Exposure`
1488  The fully ISR corrected exposure.
1489 
1490  Raises
1491  ------
1492  RuntimeError
1493  Raised if a configuration option is set to True, but the
1494  required calibration data does not exist.
1495 
1496  """
1497  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1498 
1499  ccdExposure = sensorRef.get(self.config.datasetType)
1500 
1501  camera = sensorRef.get("camera")
1502  isrData = self.readIsrData(sensorRef, ccdExposure)
1503 
1504  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1505 
1506  if self.config.doWrite:
1507  sensorRef.put(result.exposure, "postISRCCD")
1508  if result.preInterpolatedExposure is not None:
1509  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1510  if result.ossThumb is not None:
1511  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1512  if result.flattenedThumb is not None:
1513  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1514 
1515  return result
1516 
1517  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1518  """!Retrieve a calibration dataset for removing instrument signature.
1519 
1520  Parameters
1521  ----------
1522 
1523  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1524  DataRef of the detector data to find calibration datasets
1525  for.
1526  datasetType : `str`
1527  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1528  dateObs : `str`, optional
1529  Date of the observation. Used to correct butler failures
1530  when using fallback filters.
1531  immediate : `Bool`
1532  If True, disable butler proxies to enable error handling
1533  within this routine.
1534 
1535  Returns
1536  -------
1537  exposure : `lsst.afw.image.Exposure`
1538  Requested calibration frame.
1539 
1540  Raises
1541  ------
1542  RuntimeError
1543  Raised if no matching calibration frame can be found.
1544  """
1545  try:
1546  exp = dataRef.get(datasetType, immediate=immediate)
1547  except Exception as exc1:
1548  if not self.config.fallbackFilterName:
1549  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1550  try:
1551  if self.config.useFallbackDate and dateObs:
1552  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1553  dateObs=dateObs, immediate=immediate)
1554  else:
1555  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1556  except Exception as exc2:
1557  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1558  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1559  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1560 
1561  if self.config.doAssembleIsrExposures:
1562  exp = self.assembleCcd.assembleCcd(exp)
1563  return exp
1564 
1565  def ensureExposure(self, inputExp, camera, detectorNum):
1566  """Ensure that the data returned by Butler is a fully constructed exposure.
1567 
1568  ISR requires exposure-level image data for historical reasons, so if we did
1569  not recieve that from Butler, construct it from what we have, modifying the
1570  input in place.
1571 
1572  Parameters
1573  ----------
1574  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1575  `lsst.afw.image.ImageF`
1576  The input data structure obtained from Butler.
1577  camera : `lsst.afw.cameraGeom.camera`
1578  The camera associated with the image. Used to find the appropriate
1579  detector.
1580  detectorNum : `int`
1581  The detector this exposure should match.
1582 
1583  Returns
1584  -------
1585  inputExp : `lsst.afw.image.Exposure`
1586  The re-constructed exposure, with appropriate detector parameters.
1587 
1588  Raises
1589  ------
1590  TypeError
1591  Raised if the input data cannot be used to construct an exposure.
1592  """
1593  if isinstance(inputExp, afwImage.DecoratedImageU):
1594  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1595  elif isinstance(inputExp, afwImage.ImageF):
1596  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1597  elif isinstance(inputExp, afwImage.MaskedImageF):
1598  inputExp = afwImage.makeExposure(inputExp)
1599  elif isinstance(inputExp, afwImage.Exposure):
1600  pass
1601  elif inputExp is None:
1602  # Assume this will be caught by the setup if it is a problem.
1603  return inputExp
1604  else:
1605  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1606  (type(inputExp), ))
1607 
1608  if inputExp.getDetector() is None:
1609  inputExp.setDetector(camera[detectorNum])
1610 
1611  return inputExp
1612 
1613  def convertIntToFloat(self, exposure):
1614  """Convert exposure image from uint16 to float.
1615 
1616  If the exposure does not need to be converted, the input is
1617  immediately returned. For exposures that are converted to use
1618  floating point pixels, the variance is set to unity and the
1619  mask to zero.
1620 
1621  Parameters
1622  ----------
1623  exposure : `lsst.afw.image.Exposure`
1624  The raw exposure to be converted.
1625 
1626  Returns
1627  -------
1628  newexposure : `lsst.afw.image.Exposure`
1629  The input ``exposure``, converted to floating point pixels.
1630 
1631  Raises
1632  ------
1633  RuntimeError
1634  Raised if the exposure type cannot be converted to float.
1635 
1636  """
1637  if isinstance(exposure, afwImage.ExposureF):
1638  # Nothing to be done
1639  self.log.debug("Exposure already of type float.")
1640  return exposure
1641  if not hasattr(exposure, "convertF"):
1642  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1643 
1644  newexposure = exposure.convertF()
1645  newexposure.variance[:] = 1
1646  newexposure.mask[:] = 0x0
1647 
1648  return newexposure
1649 
1650  def maskAmplifier(self, ccdExposure, amp, defects):
1651  """Identify bad amplifiers, saturated and suspect pixels.
1652 
1653  Parameters
1654  ----------
1655  ccdExposure : `lsst.afw.image.Exposure`
1656  Input exposure to be masked.
1657  amp : `lsst.afw.table.AmpInfoCatalog`
1658  Catalog of parameters defining the amplifier on this
1659  exposure to mask.
1660  defects : `lsst.meas.algorithms.Defects`
1661  List of defects. Used to determine if the entire
1662  amplifier is bad.
1663 
1664  Returns
1665  -------
1666  badAmp : `Bool`
1667  If this is true, the entire amplifier area is covered by
1668  defects and unusable.
1669 
1670  """
1671  maskedImage = ccdExposure.getMaskedImage()
1672 
1673  badAmp = False
1674 
1675  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1676  # comparison with current defects definition.
1677  if defects is not None:
1678  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1679 
1680  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1681  # association with pixels in current ccdExposure).
1682  if badAmp:
1683  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1684  afwImage.PARENT)
1685  maskView = dataView.getMask()
1686  maskView |= maskView.getPlaneBitMask("BAD")
1687  del maskView
1688  return badAmp
1689 
1690  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1691  # Saturation and suspect pixels can be masked now, though.
1692  limits = dict()
1693  if self.config.doSaturation and not badAmp:
1694  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1695  if self.config.doSuspect and not badAmp:
1696  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1697  if math.isfinite(self.config.saturation):
1698  limits.update({self.config.saturatedMaskName: self.config.saturation})
1699 
1700  for maskName, maskThreshold in limits.items():
1701  if not math.isnan(maskThreshold):
1702  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1703  isrFunctions.makeThresholdMask(
1704  maskedImage=dataView,
1705  threshold=maskThreshold,
1706  growFootprints=0,
1707  maskName=maskName
1708  )
1709 
1710  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1711  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1712  afwImage.PARENT)
1713  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1714  self.config.suspectMaskName])
1715  if numpy.all(maskView.getArray() & maskVal > 0):
1716  badAmp = True
1717  maskView |= maskView.getPlaneBitMask("BAD")
1718 
1719  return badAmp
1720 
1721  def overscanCorrection(self, ccdExposure, amp):
1722  """Apply overscan correction in place.
1723 
1724  This method does initial pixel rejection of the overscan
1725  region. The overscan can also be optionally segmented to
1726  allow for discontinuous overscan responses to be fit
1727  separately. The actual overscan subtraction is performed by
1728  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1729  which is called here after the amplifier is preprocessed.
1730 
1731  Parameters
1732  ----------
1733  ccdExposure : `lsst.afw.image.Exposure`
1734  Exposure to have overscan correction performed.
1735  amp : `lsst.afw.table.AmpInfoCatalog`
1736  The amplifier to consider while correcting the overscan.
1737 
1738  Returns
1739  -------
1740  overscanResults : `lsst.pipe.base.Struct`
1741  Result struct with components:
1742  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1743  Value or fit subtracted from the amplifier image data.
1744  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1745  Value or fit subtracted from the overscan image data.
1746  - ``overscanImage`` : `lsst.afw.image.Image`
1747  Image of the overscan region with the overscan
1748  correction applied. This quantity is used to estimate
1749  the amplifier read noise empirically.
1750 
1751  Raises
1752  ------
1753  RuntimeError
1754  Raised if the ``amp`` does not contain raw pixel information.
1755 
1756  See Also
1757  --------
1758  lsst.ip.isr.isrFunctions.overscanCorrection
1759  """
1760  if not amp.getHasRawInfo():
1761  raise RuntimeError("This method must be executed on an amp with raw information.")
1762 
1763  if amp.getRawHorizontalOverscanBBox().isEmpty():
1764  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1765  return None
1766 
1767  statControl = afwMath.StatisticsControl()
1768  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1769 
1770  # Determine the bounding boxes
1771  dataBBox = amp.getRawDataBBox()
1772  oscanBBox = amp.getRawHorizontalOverscanBBox()
1773  dx0 = 0
1774  dx1 = 0
1775 
1776  prescanBBox = amp.getRawPrescanBBox()
1777  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1778  dx0 += self.config.overscanNumLeadingColumnsToSkip
1779  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1780  else:
1781  dx0 += self.config.overscanNumTrailingColumnsToSkip
1782  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1783 
1784  # Determine if we need to work on subregions of the amplifier and overscan.
1785  imageBBoxes = []
1786  overscanBBoxes = []
1787 
1788  if ((self.config.overscanBiasJump and
1789  self.config.overscanBiasJumpLocation) and
1790  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1791  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1792  self.config.overscanBiasJumpDevices)):
1793  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1794  yLower = self.config.overscanBiasJumpLocation
1795  yUpper = dataBBox.getHeight() - yLower
1796  else:
1797  yUpper = self.config.overscanBiasJumpLocation
1798  yLower = dataBBox.getHeight() - yUpper
1799 
1800  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1801  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1802  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1803  lsst.geom.Extent2I(dx0, 0),
1804  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1805  yLower)))
1806 
1807  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1808  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1809  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1810  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1811  yUpper)))
1812  else:
1813  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1814  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1815  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1816  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1817  oscanBBox.getHeight())))
1818 
1819  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1820  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1821  ampImage = ccdExposure.maskedImage[imageBBox]
1822  overscanImage = ccdExposure.maskedImage[overscanBBox]
1823 
1824  overscanArray = overscanImage.image.array
1825  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1826  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1827  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1828 
1829  statControl = afwMath.StatisticsControl()
1830  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1831 
1832  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1833  overscanImage=overscanImage,
1834  fitType=self.config.overscanFitType,
1835  order=self.config.overscanOrder,
1836  collapseRej=self.config.overscanNumSigmaClip,
1837  statControl=statControl,
1838  overscanIsInt=self.config.overscanIsInt
1839  )
1840 
1841  # Measure average overscan levels and record them in the metadata.
1842  levelStat = afwMath.MEDIAN
1843  sigmaStat = afwMath.STDEVCLIP
1844 
1845  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1846  self.config.qa.flatness.nIter)
1847  metadata = ccdExposure.getMetadata()
1848  ampNum = amp.getName()
1849  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1850  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1851  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1852  else:
1853  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1854  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1855  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1856 
1857  return overscanResults
1858 
1859  def updateVariance(self, ampExposure, amp, overscanImage=None):
1860  """Set the variance plane using the amplifier gain and read noise
1861 
1862  The read noise is calculated from the ``overscanImage`` if the
1863  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1864  the value from the amplifier data is used.
1865 
1866  Parameters
1867  ----------
1868  ampExposure : `lsst.afw.image.Exposure`
1869  Exposure to process.
1870  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1871  Amplifier detector data.
1872  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1873  Image of overscan, required only for empirical read noise.
1874 
1875  See also
1876  --------
1877  lsst.ip.isr.isrFunctions.updateVariance
1878  """
1879  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1880  gain = amp.getGain()
1881 
1882  if math.isnan(gain):
1883  gain = 1.0
1884  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1885  elif gain <= 0:
1886  patchedGain = 1.0
1887  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1888  amp.getName(), gain, patchedGain)
1889  gain = patchedGain
1890 
1891  if self.config.doEmpiricalReadNoise and overscanImage is None:
1892  self.log.info("Overscan is none for EmpiricalReadNoise.")
1893 
1894  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1895  stats = afwMath.StatisticsControl()
1896  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1897  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1898  self.log.info("Calculated empirical read noise for amp %s: %f.",
1899  amp.getName(), readNoise)
1900  else:
1901  readNoise = amp.getReadNoise()
1902 
1903  isrFunctions.updateVariance(
1904  maskedImage=ampExposure.getMaskedImage(),
1905  gain=gain,
1906  readNoise=readNoise,
1907  )
1908 
1909  def darkCorrection(self, exposure, darkExposure, invert=False):
1910  """!Apply dark correction in place.
1911 
1912  Parameters
1913  ----------
1914  exposure : `lsst.afw.image.Exposure`
1915  Exposure to process.
1916  darkExposure : `lsst.afw.image.Exposure`
1917  Dark exposure of the same size as ``exposure``.
1918  invert : `Bool`, optional
1919  If True, re-add the dark to an already corrected image.
1920 
1921  Raises
1922  ------
1923  RuntimeError
1924  Raised if either ``exposure`` or ``darkExposure`` do not
1925  have their dark time defined.
1926 
1927  See Also
1928  --------
1929  lsst.ip.isr.isrFunctions.darkCorrection
1930  """
1931  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1932  if math.isnan(expScale):
1933  raise RuntimeError("Exposure darktime is NAN.")
1934  if darkExposure.getInfo().getVisitInfo() is not None:
1935  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1936  else:
1937  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1938  # so getDarkTime() does not exist.
1939  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1940  darkScale = 1.0
1941 
1942  if math.isnan(darkScale):
1943  raise RuntimeError("Dark calib darktime is NAN.")
1944  isrFunctions.darkCorrection(
1945  maskedImage=exposure.getMaskedImage(),
1946  darkMaskedImage=darkExposure.getMaskedImage(),
1947  expScale=expScale,
1948  darkScale=darkScale,
1949  invert=invert,
1950  trimToFit=self.config.doTrimToMatchCalib
1951  )
1952 
1953  def doLinearize(self, detector):
1954  """!Check if linearization is needed for the detector cameraGeom.
1955 
1956  Checks config.doLinearize and the linearity type of the first
1957  amplifier.
1958 
1959  Parameters
1960  ----------
1961  detector : `lsst.afw.cameraGeom.Detector`
1962  Detector to get linearity type from.
1963 
1964  Returns
1965  -------
1966  doLinearize : `Bool`
1967  If True, linearization should be performed.
1968  """
1969  return self.config.doLinearize and \
1970  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
1971 
1972  def flatCorrection(self, exposure, flatExposure, invert=False):
1973  """!Apply flat correction in place.
1974 
1975  Parameters
1976  ----------
1977  exposure : `lsst.afw.image.Exposure`
1978  Exposure to process.
1979  flatExposure : `lsst.afw.image.Exposure`
1980  Flat exposure of the same size as ``exposure``.
1981  invert : `Bool`, optional
1982  If True, unflatten an already flattened image.
1983 
1984  See Also
1985  --------
1986  lsst.ip.isr.isrFunctions.flatCorrection
1987  """
1988  isrFunctions.flatCorrection(
1989  maskedImage=exposure.getMaskedImage(),
1990  flatMaskedImage=flatExposure.getMaskedImage(),
1991  scalingType=self.config.flatScalingType,
1992  userScale=self.config.flatUserScale,
1993  invert=invert,
1994  trimToFit=self.config.doTrimToMatchCalib
1995  )
1996 
1997  def saturationDetection(self, exposure, amp):
1998  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1999 
2000  Parameters
2001  ----------
2002  exposure : `lsst.afw.image.Exposure`
2003  Exposure to process. Only the amplifier DataSec is processed.
2004  amp : `lsst.afw.table.AmpInfoCatalog`
2005  Amplifier detector data.
2006 
2007  See Also
2008  --------
2009  lsst.ip.isr.isrFunctions.makeThresholdMask
2010  """
2011  if not math.isnan(amp.getSaturation()):
2012  maskedImage = exposure.getMaskedImage()
2013  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2014  isrFunctions.makeThresholdMask(
2015  maskedImage=dataView,
2016  threshold=amp.getSaturation(),
2017  growFootprints=0,
2018  maskName=self.config.saturatedMaskName,
2019  )
2020 
2021  def saturationInterpolation(self, exposure):
2022  """!Interpolate over saturated pixels, in place.
2023 
2024  This method should be called after `saturationDetection`, to
2025  ensure that the saturated pixels have been identified in the
2026  SAT mask. It should also be called after `assembleCcd`, since
2027  saturated regions may cross amplifier boundaries.
2028 
2029  Parameters
2030  ----------
2031  exposure : `lsst.afw.image.Exposure`
2032  Exposure to process.
2033 
2034  See Also
2035  --------
2036  lsst.ip.isr.isrTask.saturationDetection
2037  lsst.ip.isr.isrFunctions.interpolateFromMask
2038  """
2039  isrFunctions.interpolateFromMask(
2040  maskedImage=exposure.getMaskedImage(),
2041  fwhm=self.config.fwhm,
2042  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2043  maskNameList=list(self.config.saturatedMaskName),
2044  )
2045 
2046  def suspectDetection(self, exposure, amp):
2047  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2048 
2049  Parameters
2050  ----------
2051  exposure : `lsst.afw.image.Exposure`
2052  Exposure to process. Only the amplifier DataSec is processed.
2053  amp : `lsst.afw.table.AmpInfoCatalog`
2054  Amplifier detector data.
2055 
2056  See Also
2057  --------
2058  lsst.ip.isr.isrFunctions.makeThresholdMask
2059 
2060  Notes
2061  -----
2062  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2063  This is intended to indicate pixels that may be affected by unknown systematics;
2064  for example if non-linearity corrections above a certain level are unstable
2065  then that would be a useful value for suspectLevel. A value of `nan` indicates
2066  that no such level exists and no pixels are to be masked as suspicious.
2067  """
2068  suspectLevel = amp.getSuspectLevel()
2069  if math.isnan(suspectLevel):
2070  return
2071 
2072  maskedImage = exposure.getMaskedImage()
2073  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2074  isrFunctions.makeThresholdMask(
2075  maskedImage=dataView,
2076  threshold=suspectLevel,
2077  growFootprints=0,
2078  maskName=self.config.suspectMaskName,
2079  )
2080 
2081  def maskDefect(self, exposure, defectBaseList):
2082  """!Mask defects using mask plane "BAD", in place.
2083 
2084  Parameters
2085  ----------
2086  exposure : `lsst.afw.image.Exposure`
2087  Exposure to process.
2088  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2089  `lsst.afw.image.DefectBase`.
2090  List of defects to mask.
2091 
2092  Notes
2093  -----
2094  Call this after CCD assembly, since defects may cross amplifier boundaries.
2095  """
2096  maskedImage = exposure.getMaskedImage()
2097  if not isinstance(defectBaseList, Defects):
2098  # Promotes DefectBase to Defect
2099  defectList = Defects(defectBaseList)
2100  else:
2101  defectList = defectBaseList
2102  defectList.maskPixels(maskedImage, maskName="BAD")
2103 
2104  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2105  """!Mask edge pixels with applicable mask plane.
2106 
2107  Parameters
2108  ----------
2109  exposure : `lsst.afw.image.Exposure`
2110  Exposure to process.
2111  numEdgePixels : `int`, optional
2112  Number of edge pixels to mask.
2113  maskPlane : `str`, optional
2114  Mask plane name to use.
2115  """
2116  maskedImage = exposure.getMaskedImage()
2117  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2118 
2119  if numEdgePixels > 0:
2120  goodBBox = maskedImage.getBBox()
2121  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2122  goodBBox.grow(-numEdgePixels)
2123  # Mask pixels outside goodBBox
2124  SourceDetectionTask.setEdgeBits(
2125  maskedImage,
2126  goodBBox,
2127  maskBitMask
2128  )
2129 
2130  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2131  """Mask and interpolate defects using mask plane "BAD", in place.
2132 
2133  Parameters
2134  ----------
2135  exposure : `lsst.afw.image.Exposure`
2136  Exposure to process.
2137  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2138  `lsst.afw.image.DefectBase`.
2139  List of defects to mask and interpolate.
2140 
2141  See Also
2142  --------
2143  lsst.ip.isr.isrTask.maskDefect()
2144  """
2145  self.maskDefect(exposure, defectBaseList)
2146  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2147  maskPlane="SUSPECT")
2148  isrFunctions.interpolateFromMask(
2149  maskedImage=exposure.getMaskedImage(),
2150  fwhm=self.config.fwhm,
2151  growSaturatedFootprints=0,
2152  maskNameList=["BAD"],
2153  )
2154 
2155  def maskNan(self, exposure):
2156  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2157 
2158  Parameters
2159  ----------
2160  exposure : `lsst.afw.image.Exposure`
2161  Exposure to process.
2162 
2163  Notes
2164  -----
2165  We mask over all NaNs, including those that are masked with
2166  other bits (because those may or may not be interpolated over
2167  later, and we want to remove all NaNs). Despite this
2168  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2169  the historical name.
2170  """
2171  maskedImage = exposure.getMaskedImage()
2172 
2173  # Find and mask NaNs
2174  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2175  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2176  numNans = maskNans(maskedImage, maskVal)
2177  self.metadata.set("NUMNANS", numNans)
2178  if numNans > 0:
2179  self.log.warn("There were %d unmasked NaNs.", numNans)
2180 
2181  def maskAndInterpolateNan(self, exposure):
2182  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2183 
2184  Parameters
2185  ----------
2186  exposure : `lsst.afw.image.Exposure`
2187  Exposure to process.
2188 
2189  See Also
2190  --------
2191  lsst.ip.isr.isrTask.maskNan()
2192  """
2193  self.maskNan(exposure)
2194  isrFunctions.interpolateFromMask(
2195  maskedImage=exposure.getMaskedImage(),
2196  fwhm=self.config.fwhm,
2197  growSaturatedFootprints=0,
2198  maskNameList=["UNMASKEDNAN"],
2199  )
2200 
2201  def measureBackground(self, exposure, IsrQaConfig=None):
2202  """Measure the image background in subgrids, for quality control purposes.
2203 
2204  Parameters
2205  ----------
2206  exposure : `lsst.afw.image.Exposure`
2207  Exposure to process.
2208  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2209  Configuration object containing parameters on which background
2210  statistics and subgrids to use.
2211  """
2212  if IsrQaConfig is not None:
2213  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2214  IsrQaConfig.flatness.nIter)
2215  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2216  statsControl.setAndMask(maskVal)
2217  maskedImage = exposure.getMaskedImage()
2218  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2219  skyLevel = stats.getValue(afwMath.MEDIAN)
2220  skySigma = stats.getValue(afwMath.STDEVCLIP)
2221  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2222  metadata = exposure.getMetadata()
2223  metadata.set('SKYLEVEL', skyLevel)
2224  metadata.set('SKYSIGMA', skySigma)
2225 
2226  # calcluating flatlevel over the subgrids
2227  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2228  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2229  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2230  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2231  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2232  skyLevels = numpy.zeros((nX, nY))
2233 
2234  for j in range(nY):
2235  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2236  for i in range(nX):
2237  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2238 
2239  xLLC = xc - meshXHalf
2240  yLLC = yc - meshYHalf
2241  xURC = xc + meshXHalf - 1
2242  yURC = yc + meshYHalf - 1
2243 
2244  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2245  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2246 
2247  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2248 
2249  good = numpy.where(numpy.isfinite(skyLevels))
2250  skyMedian = numpy.median(skyLevels[good])
2251  flatness = (skyLevels[good] - skyMedian) / skyMedian
2252  flatness_rms = numpy.std(flatness)
2253  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2254 
2255  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2256  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2257  nX, nY, flatness_pp, flatness_rms)
2258 
2259  metadata.set('FLATNESS_PP', float(flatness_pp))
2260  metadata.set('FLATNESS_RMS', float(flatness_rms))
2261  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2262  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2263  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2264 
2265  def roughZeroPoint(self, exposure):
2266  """Set an approximate magnitude zero point for the exposure.
2267 
2268  Parameters
2269  ----------
2270  exposure : `lsst.afw.image.Exposure`
2271  Exposure to process.
2272  """
2273  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2274  if filterName in self.config.fluxMag0T1:
2275  fluxMag0 = self.config.fluxMag0T1[filterName]
2276  else:
2277  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2278  fluxMag0 = self.config.defaultFluxMag0T1
2279 
2280  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2281  if not expTime > 0: # handle NaN as well as <= 0
2282  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2283  return
2284 
2285  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2286  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2287 
2288  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2289  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2290 
2291  Parameters
2292  ----------
2293  ccdExposure : `lsst.afw.image.Exposure`
2294  Exposure to process.
2295  fpPolygon : `lsst.afw.geom.Polygon`
2296  Polygon in focal plane coordinates.
2297  """
2298  # Get ccd corners in focal plane coordinates
2299  ccd = ccdExposure.getDetector()
2300  fpCorners = ccd.getCorners(FOCAL_PLANE)
2301  ccdPolygon = Polygon(fpCorners)
2302 
2303  # Get intersection of ccd corners with fpPolygon
2304  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2305 
2306  # Transform back to pixel positions and build new polygon
2307  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2308  validPolygon = Polygon(ccdPoints)
2309  ccdExposure.getInfo().setValidPolygon(validPolygon)
2310 
2311  @contextmanager
2312  def flatContext(self, exp, flat, dark=None):
2313  """Context manager that applies and removes flats and darks,
2314  if the task is configured to apply them.
2315 
2316  Parameters
2317  ----------
2318  exp : `lsst.afw.image.Exposure`
2319  Exposure to process.
2320  flat : `lsst.afw.image.Exposure`
2321  Flat exposure the same size as ``exp``.
2322  dark : `lsst.afw.image.Exposure`, optional
2323  Dark exposure the same size as ``exp``.
2324 
2325  Yields
2326  ------
2327  exp : `lsst.afw.image.Exposure`
2328  The flat and dark corrected exposure.
2329  """
2330  if self.config.doDark and dark is not None:
2331  self.darkCorrection(exp, dark)
2332  if self.config.doFlat:
2333  self.flatCorrection(exp, flat)
2334  try:
2335  yield exp
2336  finally:
2337  if self.config.doFlat:
2338  self.flatCorrection(exp, flat, invert=True)
2339  if self.config.doDark and dark is not None:
2340  self.darkCorrection(exp, dark, invert=True)
2341 
2342  def debugView(self, exposure, stepname):
2343  """Utility function to examine ISR exposure at different stages.
2344 
2345  Parameters
2346  ----------
2347  exposure : `lsst.afw.image.Exposure`
2348  Exposure to view.
2349  stepname : `str`
2350  State of processing to view.
2351  """
2352  frame = getDebugFrame(self._display, stepname)
2353  if frame:
2354  display = getDisplay(frame)
2355  display.scale('asinh', 'zscale')
2356  display.mtv(exposure)
2357  prompt = "Press Enter to continue [c]... "
2358  while True:
2359  ans = input(prompt).lower()
2360  if ans in ("", "c",):
2361  break
2362 
2363 
2364 class FakeAmp(object):
2365  """A Detector-like object that supports returning gain and saturation level
2366 
2367  This is used when the input exposure does not have a detector.
2368 
2369  Parameters
2370  ----------
2371  exposure : `lsst.afw.image.Exposure`
2372  Exposure to generate a fake amplifier for.
2373  config : `lsst.ip.isr.isrTaskConfig`
2374  Configuration to apply to the fake amplifier.
2375  """
2376 
2377  def __init__(self, exposure, config):
2378  self._bbox = exposure.getBBox(afwImage.LOCAL)
2380  self._gain = config.gain
2381  self._readNoise = config.readNoise
2382  self._saturation = config.saturation
2383 
2384  def getBBox(self):
2385  return self._bbox
2386 
2387  def getRawBBox(self):
2388  return self._bbox
2389 
2390  def getHasRawInfo(self):
2391  return True # but see getRawHorizontalOverscanBBox()
2392 
2394  return self._RawHorizontalOverscanBBox
2395 
2396  def getGain(self):
2397  return self._gain
2398 
2399  def getReadNoise(self):
2400  return self._readNoise
2401 
2402  def getSaturation(self):
2403  return self._saturation
2404 
2405  def getSuspectLevel(self):
2406  return float("NaN")
2407 
2408 
2409 class RunIsrConfig(pexConfig.Config):
2410  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2411 
2412 
2413 class RunIsrTask(pipeBase.CmdLineTask):
2414  """Task to wrap the default IsrTask to allow it to be retargeted.
2415 
2416  The standard IsrTask can be called directly from a command line
2417  program, but doing so removes the ability of the task to be
2418  retargeted. As most cameras override some set of the IsrTask
2419  methods, this would remove those data-specific methods in the
2420  output post-ISR images. This wrapping class fixes the issue,
2421  allowing identical post-ISR images to be generated by both the
2422  processCcd and isrTask code.
2423  """
2424  ConfigClass = RunIsrConfig
2425  _DefaultName = "runIsr"
2426 
2427  def __init__(self, *args, **kwargs):
2428  super().__init__(*args, **kwargs)
2429  self.makeSubtask("isr")
2430 
2431  def runDataRef(self, dataRef):
2432  """
2433  Parameters
2434  ----------
2435  dataRef : `lsst.daf.persistence.ButlerDataRef`
2436  data reference of the detector data to be processed
2437 
2438  Returns
2439  -------
2440  result : `pipeBase.Struct`
2441  Result struct with component:
2442 
2443  - exposure : `lsst.afw.image.Exposure`
2444  Post-ISR processed exposure.
2445  """
2446  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1466
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2201
def debugView(self, exposure, stepname)
Definition: isrTask.py:2342
def __init__(self, kwargs)
Definition: isrTask.py:795
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1565
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:854
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2104
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:804
def runDataRef(self, dataRef)
Definition: isrTask.py:2431
def __init__(self, args, kwargs)
Definition: isrTask.py:2427
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2265
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2130
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2393
def maskNan(self, exposure)
Definition: isrTask.py:2155
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2081
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1721
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1613
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1972
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1020
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1909
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1953
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2288
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1650
def __init__(self, config=None)
Definition: isrTask.py:173
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2312
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1517
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1859
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2181
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2046
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2021
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1997
def __init__(self, exposure, config)
Definition: isrTask.py:2377