lsst.ip.isr  19.0.0-18-g2c901ce+5
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "exposure", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.Input(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "detector", "exposure"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ExposureF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ExposureF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="ExposureF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  newBFKernel = cT.PrerequisiteInput(
112  name='brighterFatterKernel',
113  doc="Newer complete kernel + gain solutions.",
114  storageClass="BrighterFatterKernel",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  defects = cT.PrerequisiteInput(
118  name='defects',
119  doc="Input defect tables.",
120  storageClass="Defects",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  opticsTransmission = cT.PrerequisiteInput(
124  name="transmission_optics",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the optics.",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = cT.PrerequisiteInput(
130  name="transmission_filter",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the filter.",
133  dimensions=["instrument", "physical_filter", "calibration_label"],
134  )
135  sensorTransmission = cT.PrerequisiteInput(
136  name="transmission_sensor",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the sensor.",
139  dimensions=["instrument", "calibration_label", "detector"],
140  )
141  atmosphereTransmission = cT.PrerequisiteInput(
142  name="transmission_atmosphere",
143  storageClass="TransmissionCurve",
144  doc="Transmission curve due to the atmosphere.",
145  dimensions=["instrument"],
146  )
147  illumMaskedImage = cT.PrerequisiteInput(
148  name="illum",
149  doc="Input illumination correction.",
150  storageClass="MaskedImageF",
151  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
152  )
153 
154  outputExposure = cT.Output(
155  name='postISRCCD',
156  doc="Output ISR processed exposure.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "exposure", "detector"],
159  )
160  preInterpExposure = cT.Output(
161  name='preInterpISRCCD',
162  doc="Output ISR processed exposure, with pixels left uninterpolated.",
163  storageClass="ExposureF",
164  dimensions=["instrument", "exposure", "detector"],
165  )
166  outputOssThumbnail = cT.Output(
167  name="OssThumb",
168  doc="Output Overscan-subtracted thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "exposure", "detector"],
171  )
172  outputFlattenedThumbnail = cT.Output(
173  name="FlattenedThumb",
174  doc="Output flat-corrected thumbnail image.",
175  storageClass="Thumbnail",
176  dimensions=["instrument", "exposure", "detector"],
177  )
178 
179  def __init__(self, *, config=None):
180  super().__init__(config=config)
181 
182  if config.doBias is not True:
183  self.prerequisiteInputs.discard("bias")
184  if config.doLinearize is not True:
185  self.prerequisiteInputs.discard("linearizer")
186  if config.doCrosstalk is not True:
187  self.prerequisiteInputs.discard("crosstalkSources")
188  if config.doBrighterFatter is not True:
189  self.prerequisiteInputs.discard("bfKernel")
190  self.prerequisiteInputs.discard("newBFKernel")
191  if config.doDefect is not True:
192  self.prerequisiteInputs.discard("defects")
193  if config.doDark is not True:
194  self.prerequisiteInputs.discard("dark")
195  if config.doFlat is not True:
196  self.prerequisiteInputs.discard("flat")
197  if config.doAttachTransmissionCurve is not True:
198  self.prerequisiteInputs.discard("opticsTransmission")
199  self.prerequisiteInputs.discard("filterTransmission")
200  self.prerequisiteInputs.discard("sensorTransmission")
201  self.prerequisiteInputs.discard("atmosphereTransmission")
202  if config.doUseOpticsTransmission is not True:
203  self.prerequisiteInputs.discard("opticsTransmission")
204  if config.doUseFilterTransmission is not True:
205  self.prerequisiteInputs.discard("filterTransmission")
206  if config.doUseSensorTransmission is not True:
207  self.prerequisiteInputs.discard("sensorTransmission")
208  if config.doUseAtmosphereTransmission is not True:
209  self.prerequisiteInputs.discard("atmosphereTransmission")
210  if config.doIlluminationCorrection is not True:
211  self.prerequisiteInputs.discard("illumMaskedImage")
212 
213  if config.doWrite is not True:
214  self.outputs.discard("outputExposure")
215  self.outputs.discard("preInterpExposure")
216  self.outputs.discard("outputFlattenedThumbnail")
217  self.outputs.discard("outputOssThumbnail")
218  if config.doSaveInterpPixels is not True:
219  self.outputs.discard("preInterpExposure")
220  if config.qa.doThumbnailOss is not True:
221  self.outputs.discard("outputOssThumbnail")
222  if config.qa.doThumbnailFlattened is not True:
223  self.outputs.discard("outputFlattenedThumbnail")
224 
225 
226 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
227  pipelineConnections=IsrTaskConnections):
228  """Configuration parameters for IsrTask.
229 
230  Items are grouped in the order in which they are executed by the task.
231  """
232  datasetType = pexConfig.Field(
233  dtype=str,
234  doc="Dataset type for input data; users will typically leave this alone, "
235  "but camera-specific ISR tasks will override it",
236  default="raw",
237  )
238 
239  fallbackFilterName = pexConfig.Field(
240  dtype=str,
241  doc="Fallback default filter name for calibrations.",
242  optional=True
243  )
244  useFallbackDate = pexConfig.Field(
245  dtype=bool,
246  doc="Pass observation date when using fallback filter.",
247  default=False,
248  )
249  expectWcs = pexConfig.Field(
250  dtype=bool,
251  default=True,
252  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
253  )
254  fwhm = pexConfig.Field(
255  dtype=float,
256  doc="FWHM of PSF in arcseconds.",
257  default=1.0,
258  )
259  qa = pexConfig.ConfigField(
260  dtype=isrQa.IsrQaConfig,
261  doc="QA related configuration options.",
262  )
263 
264  # Image conversion configuration
265  doConvertIntToFloat = pexConfig.Field(
266  dtype=bool,
267  doc="Convert integer raw images to floating point values?",
268  default=True,
269  )
270 
271  # Saturated pixel handling.
272  doSaturation = pexConfig.Field(
273  dtype=bool,
274  doc="Mask saturated pixels? NB: this is totally independent of the"
275  " interpolation option - this is ONLY setting the bits in the mask."
276  " To have them interpolated make sure doSaturationInterpolation=True",
277  default=True,
278  )
279  saturatedMaskName = pexConfig.Field(
280  dtype=str,
281  doc="Name of mask plane to use in saturation detection and interpolation",
282  default="SAT",
283  )
284  saturation = pexConfig.Field(
285  dtype=float,
286  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287  default=float("NaN"),
288  )
289  growSaturationFootprintSize = pexConfig.Field(
290  dtype=int,
291  doc="Number of pixels by which to grow the saturation footprints",
292  default=1,
293  )
294 
295  # Suspect pixel handling.
296  doSuspect = pexConfig.Field(
297  dtype=bool,
298  doc="Mask suspect pixels?",
299  default=False,
300  )
301  suspectMaskName = pexConfig.Field(
302  dtype=str,
303  doc="Name of mask plane to use for suspect pixels",
304  default="SUSPECT",
305  )
306  numEdgeSuspect = pexConfig.Field(
307  dtype=int,
308  doc="Number of edge pixels to be flagged as untrustworthy.",
309  default=0,
310  )
311 
312  # Initial masking options.
313  doSetBadRegions = pexConfig.Field(
314  dtype=bool,
315  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
316  default=True,
317  )
318  badStatistic = pexConfig.ChoiceField(
319  dtype=str,
320  doc="How to estimate the average value for BAD regions.",
321  default='MEANCLIP',
322  allowed={
323  "MEANCLIP": "Correct using the (clipped) mean of good data",
324  "MEDIAN": "Correct using the median of the good data",
325  },
326  )
327 
328  # Overscan subtraction configuration.
329  doOverscan = pexConfig.Field(
330  dtype=bool,
331  doc="Do overscan subtraction?",
332  default=True,
333  )
334  overscanFitType = pexConfig.ChoiceField(
335  dtype=str,
336  doc="The method for fitting the overscan bias level.",
337  default='MEDIAN',
338  allowed={
339  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
340  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
341  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
342  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
343  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
344  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
345  "MEAN": "Correct using the mean of the overscan region",
346  "MEANCLIP": "Correct using a clipped mean of the overscan region",
347  "MEDIAN": "Correct using the median of the overscan region",
348  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
349  },
350  )
351  overscanOrder = pexConfig.Field(
352  dtype=int,
353  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
354  "or number of spline knots if overscan fit type is a spline."),
355  default=1,
356  )
357  overscanNumSigmaClip = pexConfig.Field(
358  dtype=float,
359  doc="Rejection threshold (sigma) for collapsing overscan before fit",
360  default=3.0,
361  )
362  overscanIsInt = pexConfig.Field(
363  dtype=bool,
364  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" +
365  " and overscan.FitType=MEDIAN_PER_ROW.",
366  default=True,
367  )
368  overscanNumLeadingColumnsToSkip = pexConfig.Field(
369  dtype=int,
370  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
371  default=0,
372  )
373  overscanNumTrailingColumnsToSkip = pexConfig.Field(
374  dtype=int,
375  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
376  default=0,
377  )
378  overscanMaxDev = pexConfig.Field(
379  dtype=float,
380  doc="Maximum deviation from the median for overscan",
381  default=1000.0, check=lambda x: x > 0
382  )
383  overscanBiasJump = pexConfig.Field(
384  dtype=bool,
385  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
386  default=False,
387  )
388  overscanBiasJumpKeyword = pexConfig.Field(
389  dtype=str,
390  doc="Header keyword containing information about devices.",
391  default="NO_SUCH_KEY",
392  )
393  overscanBiasJumpDevices = pexConfig.ListField(
394  dtype=str,
395  doc="List of devices that need piecewise overscan correction.",
396  default=(),
397  )
398  overscanBiasJumpLocation = pexConfig.Field(
399  dtype=int,
400  doc="Location of bias jump along y-axis.",
401  default=0,
402  )
403 
404  # Amplifier to CCD assembly configuration
405  doAssembleCcd = pexConfig.Field(
406  dtype=bool,
407  default=True,
408  doc="Assemble amp-level exposures into a ccd-level exposure?"
409  )
410  assembleCcd = pexConfig.ConfigurableField(
411  target=AssembleCcdTask,
412  doc="CCD assembly task",
413  )
414 
415  # General calibration configuration.
416  doAssembleIsrExposures = pexConfig.Field(
417  dtype=bool,
418  default=False,
419  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
420  )
421  doTrimToMatchCalib = pexConfig.Field(
422  dtype=bool,
423  default=False,
424  doc="Trim raw data to match calibration bounding boxes?"
425  )
426 
427  # Bias subtraction.
428  doBias = pexConfig.Field(
429  dtype=bool,
430  doc="Apply bias frame correction?",
431  default=True,
432  )
433  biasDataProductName = pexConfig.Field(
434  dtype=str,
435  doc="Name of the bias data product",
436  default="bias",
437  )
438 
439  # Variance construction
440  doVariance = pexConfig.Field(
441  dtype=bool,
442  doc="Calculate variance?",
443  default=True
444  )
445  gain = pexConfig.Field(
446  dtype=float,
447  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
448  default=float("NaN"),
449  )
450  readNoise = pexConfig.Field(
451  dtype=float,
452  doc="The read noise to use if no Detector is present in the Exposure",
453  default=0.0,
454  )
455  doEmpiricalReadNoise = pexConfig.Field(
456  dtype=bool,
457  default=False,
458  doc="Calculate empirical read noise instead of value from AmpInfo data?"
459  )
460 
461  # Linearization.
462  doLinearize = pexConfig.Field(
463  dtype=bool,
464  doc="Correct for nonlinearity of the detector's response?",
465  default=True,
466  )
467 
468  # Crosstalk.
469  doCrosstalk = pexConfig.Field(
470  dtype=bool,
471  doc="Apply intra-CCD crosstalk correction?",
472  default=False,
473  )
474  doCrosstalkBeforeAssemble = pexConfig.Field(
475  dtype=bool,
476  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
477  default=False,
478  )
479  crosstalk = pexConfig.ConfigurableField(
480  target=CrosstalkTask,
481  doc="Intra-CCD crosstalk correction",
482  )
483 
484  # Masking options.
485  doDefect = pexConfig.Field(
486  dtype=bool,
487  doc="Apply correction for CCD defects, e.g. hot pixels?",
488  default=True,
489  )
490  doNanMasking = pexConfig.Field(
491  dtype=bool,
492  doc="Mask NAN pixels?",
493  default=True,
494  )
495  doWidenSaturationTrails = pexConfig.Field(
496  dtype=bool,
497  doc="Widen bleed trails based on their width?",
498  default=True
499  )
500 
501  # Brighter-Fatter correction.
502  doBrighterFatter = pexConfig.Field(
503  dtype=bool,
504  default=False,
505  doc="Apply the brighter fatter correction"
506  )
507  brighterFatterLevel = pexConfig.ChoiceField(
508  dtype=str,
509  default="DETECTOR",
510  doc="The level at which to correct for brighter-fatter.",
511  allowed={
512  "AMP": "Every amplifier treated separately.",
513  "DETECTOR": "One kernel per detector",
514  }
515  )
516  brighterFatterMaxIter = pexConfig.Field(
517  dtype=int,
518  default=10,
519  doc="Maximum number of iterations for the brighter fatter correction"
520  )
521  brighterFatterThreshold = pexConfig.Field(
522  dtype=float,
523  default=1000,
524  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
525  " absolute value of the difference between the current corrected image and the one"
526  " from the previous iteration summed over all the pixels."
527  )
528  brighterFatterApplyGain = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Should the gain be applied when applying the brighter fatter correction?"
532  )
533  brighterFatterMaskGrowSize = pexConfig.Field(
534  dtype=int,
535  default=0,
536  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
537  " when brighter-fatter correction is applied."
538  )
539 
540  # Dark subtraction.
541  doDark = pexConfig.Field(
542  dtype=bool,
543  doc="Apply dark frame correction?",
544  default=True,
545  )
546  darkDataProductName = pexConfig.Field(
547  dtype=str,
548  doc="Name of the dark data product",
549  default="dark",
550  )
551 
552  # Camera-specific stray light removal.
553  doStrayLight = pexConfig.Field(
554  dtype=bool,
555  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
556  default=False,
557  )
558  strayLight = pexConfig.ConfigurableField(
559  target=StrayLightTask,
560  doc="y-band stray light correction"
561  )
562 
563  # Flat correction.
564  doFlat = pexConfig.Field(
565  dtype=bool,
566  doc="Apply flat field correction?",
567  default=True,
568  )
569  flatDataProductName = pexConfig.Field(
570  dtype=str,
571  doc="Name of the flat data product",
572  default="flat",
573  )
574  flatScalingType = pexConfig.ChoiceField(
575  dtype=str,
576  doc="The method for scaling the flat on the fly.",
577  default='USER',
578  allowed={
579  "USER": "Scale by flatUserScale",
580  "MEAN": "Scale by the inverse of the mean",
581  "MEDIAN": "Scale by the inverse of the median",
582  },
583  )
584  flatUserScale = pexConfig.Field(
585  dtype=float,
586  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
587  default=1.0,
588  )
589  doTweakFlat = pexConfig.Field(
590  dtype=bool,
591  doc="Tweak flats to match observed amplifier ratios?",
592  default=False
593  )
594 
595  # Amplifier normalization based on gains instead of using flats configuration.
596  doApplyGains = pexConfig.Field(
597  dtype=bool,
598  doc="Correct the amplifiers for their gains instead of applying flat correction",
599  default=False,
600  )
601  normalizeGains = pexConfig.Field(
602  dtype=bool,
603  doc="Normalize all the amplifiers in each CCD to have the same median value.",
604  default=False,
605  )
606 
607  # Fringe correction.
608  doFringe = pexConfig.Field(
609  dtype=bool,
610  doc="Apply fringe correction?",
611  default=True,
612  )
613  fringe = pexConfig.ConfigurableField(
614  target=FringeTask,
615  doc="Fringe subtraction task",
616  )
617  fringeAfterFlat = pexConfig.Field(
618  dtype=bool,
619  doc="Do fringe subtraction after flat-fielding?",
620  default=True,
621  )
622 
623  # Distortion model application.
624  doAddDistortionModel = pexConfig.Field(
625  dtype=bool,
626  doc="Apply a distortion model based on camera geometry to the WCS?",
627  default=True,
628  deprecated=("Camera geometry is incorporated when reading the raw files."
629  " This option no longer is used, and will be removed after v19.")
630  )
631 
632  # Initial CCD-level background statistics options.
633  doMeasureBackground = pexConfig.Field(
634  dtype=bool,
635  doc="Measure the background level on the reduced image?",
636  default=False,
637  )
638 
639  # Camera-specific masking configuration.
640  doCameraSpecificMasking = pexConfig.Field(
641  dtype=bool,
642  doc="Mask camera-specific bad regions?",
643  default=False,
644  )
645  masking = pexConfig.ConfigurableField(
646  target=MaskingTask,
647  doc="Masking task."
648  )
649 
650  # Interpolation options.
651 
652  doInterpolate = pexConfig.Field(
653  dtype=bool,
654  doc="Interpolate masked pixels?",
655  default=True,
656  )
657  doSaturationInterpolation = pexConfig.Field(
658  dtype=bool,
659  doc="Perform interpolation over pixels masked as saturated?"
660  " NB: This is independent of doSaturation; if that is False this plane"
661  " will likely be blank, resulting in a no-op here.",
662  default=True,
663  )
664  doNanInterpolation = pexConfig.Field(
665  dtype=bool,
666  doc="Perform interpolation over pixels masked as NaN?"
667  " NB: This is independent of doNanMasking; if that is False this plane"
668  " will likely be blank, resulting in a no-op here.",
669  default=True,
670  )
671  doNanInterpAfterFlat = pexConfig.Field(
672  dtype=bool,
673  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
674  "also have to interpolate them before flat-fielding."),
675  default=False,
676  )
677  maskListToInterpolate = pexConfig.ListField(
678  dtype=str,
679  doc="List of mask planes that should be interpolated.",
680  default=['SAT', 'BAD', 'UNMASKEDNAN'],
681  )
682  doSaveInterpPixels = pexConfig.Field(
683  dtype=bool,
684  doc="Save a copy of the pre-interpolated pixel values?",
685  default=False,
686  )
687 
688  # Default photometric calibration options.
689  fluxMag0T1 = pexConfig.DictField(
690  keytype=str,
691  itemtype=float,
692  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
693  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
694  ))
695  )
696  defaultFluxMag0T1 = pexConfig.Field(
697  dtype=float,
698  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
699  default=pow(10.0, 0.4*28.0)
700  )
701 
702  # Vignette correction configuration.
703  doVignette = pexConfig.Field(
704  dtype=bool,
705  doc="Apply vignetting parameters?",
706  default=False,
707  )
708  vignette = pexConfig.ConfigurableField(
709  target=VignetteTask,
710  doc="Vignetting task.",
711  )
712 
713  # Transmission curve configuration.
714  doAttachTransmissionCurve = pexConfig.Field(
715  dtype=bool,
716  default=False,
717  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
718  )
719  doUseOpticsTransmission = pexConfig.Field(
720  dtype=bool,
721  default=True,
722  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
723  )
724  doUseFilterTransmission = pexConfig.Field(
725  dtype=bool,
726  default=True,
727  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
728  )
729  doUseSensorTransmission = pexConfig.Field(
730  dtype=bool,
731  default=True,
732  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
733  )
734  doUseAtmosphereTransmission = pexConfig.Field(
735  dtype=bool,
736  default=True,
737  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
738  )
739 
740  # Illumination correction.
741  doIlluminationCorrection = pexConfig.Field(
742  dtype=bool,
743  default=False,
744  doc="Perform illumination correction?"
745  )
746  illuminationCorrectionDataProductName = pexConfig.Field(
747  dtype=str,
748  doc="Name of the illumination correction data product.",
749  default="illumcor",
750  )
751  illumScale = pexConfig.Field(
752  dtype=float,
753  doc="Scale factor for the illumination correction.",
754  default=1.0,
755  )
756  illumFilters = pexConfig.ListField(
757  dtype=str,
758  default=[],
759  doc="Only perform illumination correction for these filters."
760  )
761 
762  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
763  doWrite = pexConfig.Field(
764  dtype=bool,
765  doc="Persist postISRCCD?",
766  default=True,
767  )
768 
769  def validate(self):
770  super().validate()
771  if self.doFlat and self.doApplyGains:
772  raise ValueError("You may not specify both doFlat and doApplyGains")
773  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
774  self.config.maskListToInterpolate.append("SAT")
775  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
776  self.config.maskListToInterpolate.append("UNMASKEDNAN")
777 
778 
779 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
780  """Apply common instrument signature correction algorithms to a raw frame.
781 
782  The process for correcting imaging data is very similar from
783  camera to camera. This task provides a vanilla implementation of
784  doing these corrections, including the ability to turn certain
785  corrections off if they are not needed. The inputs to the primary
786  method, `run()`, are a raw exposure to be corrected and the
787  calibration data products. The raw input is a single chip sized
788  mosaic of all amps including overscans and other non-science
789  pixels. The method `runDataRef()` identifies and defines the
790  calibration data products, and is intended for use by a
791  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
792  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
793  subclassed for different camera, although the most camera specific
794  methods have been split into subtasks that can be redirected
795  appropriately.
796 
797  The __init__ method sets up the subtasks for ISR processing, using
798  the defaults from `lsst.ip.isr`.
799 
800  Parameters
801  ----------
802  args : `list`
803  Positional arguments passed to the Task constructor. None used at this time.
804  kwargs : `dict`, optional
805  Keyword arguments passed on to the Task constructor. None used at this time.
806  """
807  ConfigClass = IsrTaskConfig
808  _DefaultName = "isr"
809 
810  def __init__(self, **kwargs):
811  super().__init__(**kwargs)
812  self.makeSubtask("assembleCcd")
813  self.makeSubtask("crosstalk")
814  self.makeSubtask("strayLight")
815  self.makeSubtask("fringe")
816  self.makeSubtask("masking")
817  self.makeSubtask("vignette")
818 
819  def runQuantum(self, butlerQC, inputRefs, outputRefs):
820  inputs = butlerQC.get(inputRefs)
821 
822  try:
823  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
824  except Exception as e:
825  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
826  (inputRefs, e))
827 
828  inputs['isGen3'] = True
829 
830  detector = inputs['ccdExposure'].getDetector()
831 
832  if self.doLinearize(detector) is True:
833  if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
834  linearizer = linearize.Linearizer(detector=detector, log=self.log)
835  linearizer.fromYaml(inputs['linearizer'])
836  else:
837  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
838  log=self.log)
839  inputs['linearizer'] = linearizer
840 
841  if self.config.doDefect is True:
842  if "defects" in inputs and inputs['defects'] is not None:
843  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
844  # masking expects a list of defects defined by their bounding box
845  if not isinstance(inputs["defects"], Defects):
846  inputs["defects"] = Defects.fromTable(inputs["defects"])
847 
848  # Load the correct style of brighter fatter kernel, and repack
849  # the information as a numpy array.
850  if self.config.doBrighterFatter:
851  brighterFatterKernel = inputs.pop('newBFKernel', None)
852  if brighterFatterKernel is None:
853  brighterFatterKernel = inputs.get('bfKernel', None)
854 
855  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
856  detId = detector.getId()
857  inputs['bfGains'] = brighterFatterKernel.gain
858  # If the kernel is not an ndarray, it's the cp_pipe version
859  # so extract the kernel for this detector, or raise an error
860  if self.config.brighterFatterLevel == 'DETECTOR':
861  if brighterFatterKernel.detectorKernel:
862  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
863  elif brighterFatterKernel.detectorKernelFromAmpKernels:
864  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
865  else:
866  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
867  else:
868  # TODO DM-15631 for implementing this
869  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
870 
871  # Broken: DM-17169
872  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
873  # fixed for non-HSC cameras in the future.
874  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
875  # if self.config.doCrosstalk else None)
876 
877  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
878  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
879  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
880  expId=expId,
881  assembler=self.assembleCcd
882  if self.config.doAssembleIsrExposures else None)
883  else:
884  inputs['fringes'] = pipeBase.Struct(fringes=None)
885 
886  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
887  if 'strayLightData' not in inputs:
888  inputs['strayLightData'] = None
889 
890  outputs = self.run(**inputs)
891  butlerQC.put(outputs, outputRefs)
892 
893  def readIsrData(self, dataRef, rawExposure):
894  """!Retrieve necessary frames for instrument signature removal.
895 
896  Pre-fetching all required ISR data products limits the IO
897  required by the ISR. Any conflict between the calibration data
898  available and that needed for ISR is also detected prior to
899  doing processing, allowing it to fail quickly.
900 
901  Parameters
902  ----------
903  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
904  Butler reference of the detector data to be processed
905  rawExposure : `afw.image.Exposure`
906  The raw exposure that will later be corrected with the
907  retrieved calibration data; should not be modified in this
908  method.
909 
910  Returns
911  -------
912  result : `lsst.pipe.base.Struct`
913  Result struct with components (which may be `None`):
914  - ``bias``: bias calibration frame (`afw.image.Exposure`)
915  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
916  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
917  - ``dark``: dark calibration frame (`afw.image.Exposure`)
918  - ``flat``: flat calibration frame (`afw.image.Exposure`)
919  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
920  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
921  - ``fringes``: `lsst.pipe.base.Struct` with components:
922  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
923  - ``seed``: random seed derived from the ccdExposureId for random
924  number generator (`uint32`).
925  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
926  A ``TransmissionCurve`` that represents the throughput of the optics,
927  to be evaluated in focal-plane coordinates.
928  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
929  A ``TransmissionCurve`` that represents the throughput of the filter
930  itself, to be evaluated in focal-plane coordinates.
931  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
932  A ``TransmissionCurve`` that represents the throughput of the sensor
933  itself, to be evaluated in post-assembly trimmed detector coordinates.
934  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
935  A ``TransmissionCurve`` that represents the throughput of the
936  atmosphere, assumed to be spatially constant.
937  - ``strayLightData`` : `object`
938  An opaque object containing calibration information for
939  stray-light correction. If `None`, no correction will be
940  performed.
941  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
942 
943  Raises
944  ------
945  NotImplementedError :
946  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
947  """
948  try:
949  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
950  dateObs = dateObs.toPython().isoformat()
951  except RuntimeError:
952  self.log.warn("Unable to identify dateObs for rawExposure.")
953  dateObs = None
954 
955  ccd = rawExposure.getDetector()
956  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
957  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
958  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
959  if self.config.doBias else None)
960  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
961  linearizer = (dataRef.get("linearizer", immediate=True)
962  if self.doLinearize(ccd) else None)
963  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
964  linearizer.log = self.log
965  if isinstance(linearizer, numpy.ndarray):
966  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
967  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
968  if self.config.doCrosstalk else None)
969  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
970  if self.config.doDark else None)
971  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
972  dateObs=dateObs)
973  if self.config.doFlat else None)
974 
975  brighterFatterKernel = None
976  brighterFatterGains = None
977  if self.config.doBrighterFatter is True:
978  try:
979  # Use the new-style cp_pipe version of the kernel if it exists
980  # If using a new-style kernel, always use the self-consistent
981  # gains, i.e. the ones inside the kernel object itself
982  brighterFatterKernel = dataRef.get("brighterFatterKernel")
983  brighterFatterGains = brighterFatterKernel.gain
984  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
985  except NoResults:
986  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
987  brighterFatterKernel = dataRef.get("bfKernel")
988  self.log.info("Old style bright-fatter kernel (np.array) loaded")
989  except NoResults:
990  brighterFatterKernel = None
991  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
992  # If the kernel is not an ndarray, it's the cp_pipe version
993  # so extract the kernel for this detector, or raise an error
994  if self.config.brighterFatterLevel == 'DETECTOR':
995  if brighterFatterKernel.detectorKernel:
996  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
997  elif brighterFatterKernel.detectorKernelFromAmpKernels:
998  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
999  else:
1000  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1001  else:
1002  # TODO DM-15631 for implementing this
1003  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1004 
1005  defectList = (dataRef.get("defects")
1006  if self.config.doDefect else None)
1007  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1008  if self.config.doAssembleIsrExposures else None)
1009  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1010  else pipeBase.Struct(fringes=None))
1011 
1012  if self.config.doAttachTransmissionCurve:
1013  opticsTransmission = (dataRef.get("transmission_optics")
1014  if self.config.doUseOpticsTransmission else None)
1015  filterTransmission = (dataRef.get("transmission_filter")
1016  if self.config.doUseFilterTransmission else None)
1017  sensorTransmission = (dataRef.get("transmission_sensor")
1018  if self.config.doUseSensorTransmission else None)
1019  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1020  if self.config.doUseAtmosphereTransmission else None)
1021  else:
1022  opticsTransmission = None
1023  filterTransmission = None
1024  sensorTransmission = None
1025  atmosphereTransmission = None
1026 
1027  if self.config.doStrayLight:
1028  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1029  else:
1030  strayLightData = None
1031 
1032  illumMaskedImage = (self.getIsrExposure(dataRef,
1033  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1034  if (self.config.doIlluminationCorrection and
1035  filterName in self.config.illumFilters)
1036  else None)
1037 
1038  # Struct should include only kwargs to run()
1039  return pipeBase.Struct(bias=biasExposure,
1040  linearizer=linearizer,
1041  crosstalkSources=crosstalkSources,
1042  dark=darkExposure,
1043  flat=flatExposure,
1044  bfKernel=brighterFatterKernel,
1045  bfGains=brighterFatterGains,
1046  defects=defectList,
1047  fringes=fringeStruct,
1048  opticsTransmission=opticsTransmission,
1049  filterTransmission=filterTransmission,
1050  sensorTransmission=sensorTransmission,
1051  atmosphereTransmission=atmosphereTransmission,
1052  strayLightData=strayLightData,
1053  illumMaskedImage=illumMaskedImage
1054  )
1055 
1056  @pipeBase.timeMethod
1057  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1058  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1059  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1060  sensorTransmission=None, atmosphereTransmission=None,
1061  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1062  isGen3=False,
1063  ):
1064  """!Perform instrument signature removal on an exposure.
1065 
1066  Steps included in the ISR processing, in order performed, are:
1067  - saturation and suspect pixel masking
1068  - overscan subtraction
1069  - CCD assembly of individual amplifiers
1070  - bias subtraction
1071  - variance image construction
1072  - linearization of non-linear response
1073  - crosstalk masking
1074  - brighter-fatter correction
1075  - dark subtraction
1076  - fringe correction
1077  - stray light subtraction
1078  - flat correction
1079  - masking of known defects and camera specific features
1080  - vignette calculation
1081  - appending transmission curve and distortion model
1082 
1083  Parameters
1084  ----------
1085  ccdExposure : `lsst.afw.image.Exposure`
1086  The raw exposure that is to be run through ISR. The
1087  exposure is modified by this method.
1088  camera : `lsst.afw.cameraGeom.Camera`, optional
1089  The camera geometry for this exposure. Used to select the
1090  distortion model appropriate for this data.
1091  bias : `lsst.afw.image.Exposure`, optional
1092  Bias calibration frame.
1093  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1094  Functor for linearization.
1095  crosstalkSources : `list`, optional
1096  List of possible crosstalk sources.
1097  dark : `lsst.afw.image.Exposure`, optional
1098  Dark calibration frame.
1099  flat : `lsst.afw.image.Exposure`, optional
1100  Flat calibration frame.
1101  bfKernel : `numpy.ndarray`, optional
1102  Brighter-fatter kernel.
1103  bfGains : `dict` of `float`, optional
1104  Gains used to override the detector's nominal gains for the
1105  brighter-fatter correction. A dict keyed by amplifier name for
1106  the detector in question.
1107  defects : `lsst.meas.algorithms.Defects`, optional
1108  List of defects.
1109  fringes : `lsst.pipe.base.Struct`, optional
1110  Struct containing the fringe correction data, with
1111  elements:
1112  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1113  - ``seed``: random seed derived from the ccdExposureId for random
1114  number generator (`uint32`)
1115  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1116  A ``TransmissionCurve`` that represents the throughput of the optics,
1117  to be evaluated in focal-plane coordinates.
1118  filterTransmission : `lsst.afw.image.TransmissionCurve`
1119  A ``TransmissionCurve`` that represents the throughput of the filter
1120  itself, to be evaluated in focal-plane coordinates.
1121  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1122  A ``TransmissionCurve`` that represents the throughput of the sensor
1123  itself, to be evaluated in post-assembly trimmed detector coordinates.
1124  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1125  A ``TransmissionCurve`` that represents the throughput of the
1126  atmosphere, assumed to be spatially constant.
1127  detectorNum : `int`, optional
1128  The integer number for the detector to process.
1129  isGen3 : bool, optional
1130  Flag this call to run() as using the Gen3 butler environment.
1131  strayLightData : `object`, optional
1132  Opaque object containing calibration information for stray-light
1133  correction. If `None`, no correction will be performed.
1134  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1135  Illumination correction image.
1136 
1137  Returns
1138  -------
1139  result : `lsst.pipe.base.Struct`
1140  Result struct with component:
1141  - ``exposure`` : `afw.image.Exposure`
1142  The fully ISR corrected exposure.
1143  - ``outputExposure`` : `afw.image.Exposure`
1144  An alias for `exposure`
1145  - ``ossThumb`` : `numpy.ndarray`
1146  Thumbnail image of the exposure after overscan subtraction.
1147  - ``flattenedThumb`` : `numpy.ndarray`
1148  Thumbnail image of the exposure after flat-field correction.
1149 
1150  Raises
1151  ------
1152  RuntimeError
1153  Raised if a configuration option is set to True, but the
1154  required calibration data has not been specified.
1155 
1156  Notes
1157  -----
1158  The current processed exposure can be viewed by setting the
1159  appropriate lsstDebug entries in the `debug.display`
1160  dictionary. The names of these entries correspond to some of
1161  the IsrTaskConfig Boolean options, with the value denoting the
1162  frame to use. The exposure is shown inside the matching
1163  option check and after the processing of that step has
1164  finished. The steps with debug points are:
1165 
1166  doAssembleCcd
1167  doBias
1168  doCrosstalk
1169  doBrighterFatter
1170  doDark
1171  doFringe
1172  doStrayLight
1173  doFlat
1174 
1175  In addition, setting the "postISRCCD" entry displays the
1176  exposure after all ISR processing has finished.
1177 
1178  """
1179 
1180  if isGen3 is True:
1181  # Gen3 currently cannot automatically do configuration overrides.
1182  # DM-15257 looks to discuss this issue.
1183  # Configure input exposures;
1184  if detectorNum is None:
1185  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1186 
1187  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1188  bias = self.ensureExposure(bias, camera, detectorNum)
1189  dark = self.ensureExposure(dark, camera, detectorNum)
1190  flat = self.ensureExposure(flat, camera, detectorNum)
1191  else:
1192  if isinstance(ccdExposure, ButlerDataRef):
1193  return self.runDataRef(ccdExposure)
1194 
1195  ccd = ccdExposure.getDetector()
1196  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1197 
1198  if not ccd:
1199  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1200  ccd = [FakeAmp(ccdExposure, self.config)]
1201 
1202  # Validate Input
1203  if self.config.doBias and bias is None:
1204  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1205  if self.doLinearize(ccd) and linearizer is None:
1206  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1207  if self.config.doBrighterFatter and bfKernel is None:
1208  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1209  if self.config.doDark and dark is None:
1210  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1211  if self.config.doFlat and flat is None:
1212  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1213  if self.config.doDefect and defects is None:
1214  raise RuntimeError("Must supply defects if config.doDefect=True.")
1215  if (self.config.doFringe and filterName in self.fringe.config.filters and
1216  fringes.fringes is None):
1217  # The `fringes` object needs to be a pipeBase.Struct, as
1218  # we use it as a `dict` for the parameters of
1219  # `FringeTask.run()`. The `fringes.fringes` `list` may
1220  # not be `None` if `doFringe=True`. Otherwise, raise.
1221  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1222  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1223  illumMaskedImage is None):
1224  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1225 
1226  # Begin ISR processing.
1227  if self.config.doConvertIntToFloat:
1228  self.log.info("Converting exposure to floating point values.")
1229  ccdExposure = self.convertIntToFloat(ccdExposure)
1230 
1231  # Amplifier level processing.
1232  overscans = []
1233  for amp in ccd:
1234  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1235  if ccdExposure.getBBox().contains(amp.getBBox()):
1236  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1237  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1238 
1239  if self.config.doOverscan and not badAmp:
1240  # Overscan correction on amp-by-amp basis.
1241  overscanResults = self.overscanCorrection(ccdExposure, amp)
1242  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1243  if overscanResults is not None and \
1244  self.config.qa is not None and self.config.qa.saveStats is True:
1245  if isinstance(overscanResults.overscanFit, float):
1246  qaMedian = overscanResults.overscanFit
1247  qaStdev = float("NaN")
1248  else:
1249  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1250  afwMath.MEDIAN | afwMath.STDEVCLIP)
1251  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1252  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1253 
1254  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1255  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1256  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1257  amp.getName(), qaMedian, qaStdev)
1258  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1259  else:
1260  if badAmp:
1261  self.log.warn("Amplifier %s is bad.", amp.getName())
1262  overscanResults = None
1263 
1264  overscans.append(overscanResults if overscanResults is not None else None)
1265  else:
1266  self.log.info("Skipped OSCAN for %s.", amp.getName())
1267 
1268  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1269  self.log.info("Applying crosstalk correction.")
1270  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1271  self.debugView(ccdExposure, "doCrosstalk")
1272 
1273  if self.config.doAssembleCcd:
1274  self.log.info("Assembling CCD from amplifiers.")
1275  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1276 
1277  if self.config.expectWcs and not ccdExposure.getWcs():
1278  self.log.warn("No WCS found in input exposure.")
1279  self.debugView(ccdExposure, "doAssembleCcd")
1280 
1281  ossThumb = None
1282  if self.config.qa.doThumbnailOss:
1283  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1284 
1285  if self.config.doBias:
1286  self.log.info("Applying bias correction.")
1287  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1288  trimToFit=self.config.doTrimToMatchCalib)
1289  self.debugView(ccdExposure, "doBias")
1290 
1291  if self.config.doVariance:
1292  for amp, overscanResults in zip(ccd, overscans):
1293  if ccdExposure.getBBox().contains(amp.getBBox()):
1294  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1295  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1296  if overscanResults is not None:
1297  self.updateVariance(ampExposure, amp,
1298  overscanImage=overscanResults.overscanImage)
1299  else:
1300  self.updateVariance(ampExposure, amp,
1301  overscanImage=None)
1302  if self.config.qa is not None and self.config.qa.saveStats is True:
1303  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1304  afwMath.MEDIAN | afwMath.STDEVCLIP)
1305  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1306  qaStats.getValue(afwMath.MEDIAN))
1307  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1308  qaStats.getValue(afwMath.STDEVCLIP))
1309  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1310  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1311  qaStats.getValue(afwMath.STDEVCLIP))
1312 
1313  if self.doLinearize(ccd):
1314  self.log.info("Applying linearizer.")
1315  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1316  detector=ccd, log=self.log)
1317 
1318  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1319  self.log.info("Applying crosstalk correction.")
1320  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1321  self.debugView(ccdExposure, "doCrosstalk")
1322 
1323  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1324  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1325  if self.config.doDefect:
1326  self.log.info("Masking defects.")
1327  self.maskDefect(ccdExposure, defects)
1328 
1329  if self.config.numEdgeSuspect > 0:
1330  self.log.info("Masking edges as SUSPECT.")
1331  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1332  maskPlane="SUSPECT")
1333 
1334  if self.config.doNanMasking:
1335  self.log.info("Masking NAN value pixels.")
1336  self.maskNan(ccdExposure)
1337 
1338  if self.config.doWidenSaturationTrails:
1339  self.log.info("Widening saturation trails.")
1340  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1341 
1342  if self.config.doCameraSpecificMasking:
1343  self.log.info("Masking regions for camera specific reasons.")
1344  self.masking.run(ccdExposure)
1345 
1346  if self.config.doBrighterFatter:
1347  # We need to apply flats and darks before we can interpolate, and we
1348  # need to interpolate before we do B-F, but we do B-F without the
1349  # flats and darks applied so we can work in units of electrons or holes.
1350  # This context manager applies and then removes the darks and flats.
1351  #
1352  # We also do not want to interpolate values here, so operate on temporary
1353  # images so we can apply only the BF-correction and roll back the
1354  # interpolation.
1355  interpExp = ccdExposure.clone()
1356  with self.flatContext(interpExp, flat, dark):
1357  isrFunctions.interpolateFromMask(
1358  maskedImage=interpExp.getMaskedImage(),
1359  fwhm=self.config.fwhm,
1360  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1361  maskNameList=self.config.maskListToInterpolate
1362  )
1363  bfExp = interpExp.clone()
1364 
1365  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1366  type(bfKernel), type(bfGains))
1367  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1368  self.config.brighterFatterMaxIter,
1369  self.config.brighterFatterThreshold,
1370  self.config.brighterFatterApplyGain,
1371  bfGains)
1372  if bfResults[1] == self.config.brighterFatterMaxIter:
1373  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1374  bfResults[0])
1375  else:
1376  self.log.info("Finished brighter fatter correction in %d iterations.",
1377  bfResults[1])
1378  image = ccdExposure.getMaskedImage().getImage()
1379  bfCorr = bfExp.getMaskedImage().getImage()
1380  bfCorr -= interpExp.getMaskedImage().getImage()
1381  image += bfCorr
1382 
1383  # Applying the brighter-fatter correction applies a
1384  # convolution to the science image. At the edges this
1385  # convolution may not have sufficient valid pixels to
1386  # produce a valid correction. Mark pixels within the size
1387  # of the brighter-fatter kernel as EDGE to warn of this
1388  # fact.
1389  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1390  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1391  maskPlane="EDGE")
1392 
1393  if self.config.brighterFatterMaskGrowSize > 0:
1394  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1395  for maskPlane in self.config.maskListToInterpolate:
1396  isrFunctions.growMasks(ccdExposure.getMask(),
1397  radius=self.config.brighterFatterMaskGrowSize,
1398  maskNameList=maskPlane,
1399  maskValue=maskPlane)
1400 
1401  self.debugView(ccdExposure, "doBrighterFatter")
1402 
1403  if self.config.doDark:
1404  self.log.info("Applying dark correction.")
1405  self.darkCorrection(ccdExposure, dark)
1406  self.debugView(ccdExposure, "doDark")
1407 
1408  if self.config.doFringe and not self.config.fringeAfterFlat:
1409  self.log.info("Applying fringe correction before flat.")
1410  self.fringe.run(ccdExposure, **fringes.getDict())
1411  self.debugView(ccdExposure, "doFringe")
1412 
1413  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1414  self.log.info("Checking strayLight correction.")
1415  self.strayLight.run(ccdExposure, strayLightData)
1416  self.debugView(ccdExposure, "doStrayLight")
1417 
1418  if self.config.doFlat:
1419  self.log.info("Applying flat correction.")
1420  self.flatCorrection(ccdExposure, flat)
1421  self.debugView(ccdExposure, "doFlat")
1422 
1423  if self.config.doApplyGains:
1424  self.log.info("Applying gain correction instead of flat.")
1425  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1426 
1427  if self.config.doFringe and self.config.fringeAfterFlat:
1428  self.log.info("Applying fringe correction after flat.")
1429  self.fringe.run(ccdExposure, **fringes.getDict())
1430 
1431  if self.config.doVignette:
1432  self.log.info("Constructing Vignette polygon.")
1433  self.vignettePolygon = self.vignette.run(ccdExposure)
1434 
1435  if self.config.vignette.doWriteVignettePolygon:
1436  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1437 
1438  if self.config.doAttachTransmissionCurve:
1439  self.log.info("Adding transmission curves.")
1440  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1441  filterTransmission=filterTransmission,
1442  sensorTransmission=sensorTransmission,
1443  atmosphereTransmission=atmosphereTransmission)
1444 
1445  flattenedThumb = None
1446  if self.config.qa.doThumbnailFlattened:
1447  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1448 
1449  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1450  self.log.info("Performing illumination correction.")
1451  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1452  illumMaskedImage, illumScale=self.config.illumScale,
1453  trimToFit=self.config.doTrimToMatchCalib)
1454 
1455  preInterpExp = None
1456  if self.config.doSaveInterpPixels:
1457  preInterpExp = ccdExposure.clone()
1458 
1459  # Reset and interpolate bad pixels.
1460  #
1461  # Large contiguous bad regions (which should have the BAD mask
1462  # bit set) should have their values set to the image median.
1463  # This group should include defects and bad amplifiers. As the
1464  # area covered by these defects are large, there's little
1465  # reason to expect that interpolation would provide a more
1466  # useful value.
1467  #
1468  # Smaller defects can be safely interpolated after the larger
1469  # regions have had their pixel values reset. This ensures
1470  # that the remaining defects adjacent to bad amplifiers (as an
1471  # example) do not attempt to interpolate extreme values.
1472  if self.config.doSetBadRegions:
1473  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1474  if badPixelCount > 0:
1475  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1476 
1477  if self.config.doInterpolate:
1478  self.log.info("Interpolating masked pixels.")
1479  isrFunctions.interpolateFromMask(
1480  maskedImage=ccdExposure.getMaskedImage(),
1481  fwhm=self.config.fwhm,
1482  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1483  maskNameList=list(self.config.maskListToInterpolate)
1484  )
1485 
1486  self.roughZeroPoint(ccdExposure)
1487 
1488  if self.config.doMeasureBackground:
1489  self.log.info("Measuring background level.")
1490  self.measureBackground(ccdExposure, self.config.qa)
1491 
1492  if self.config.qa is not None and self.config.qa.saveStats is True:
1493  for amp in ccd:
1494  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1495  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1496  afwMath.MEDIAN | afwMath.STDEVCLIP)
1497  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1498  qaStats.getValue(afwMath.MEDIAN))
1499  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1500  qaStats.getValue(afwMath.STDEVCLIP))
1501  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1502  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1503  qaStats.getValue(afwMath.STDEVCLIP))
1504 
1505  self.debugView(ccdExposure, "postISRCCD")
1506 
1507  return pipeBase.Struct(
1508  exposure=ccdExposure,
1509  ossThumb=ossThumb,
1510  flattenedThumb=flattenedThumb,
1511 
1512  preInterpolatedExposure=preInterpExp,
1513  outputExposure=ccdExposure,
1514  outputOssThumbnail=ossThumb,
1515  outputFlattenedThumbnail=flattenedThumb,
1516  )
1517 
1518  @pipeBase.timeMethod
1519  def runDataRef(self, sensorRef):
1520  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1521 
1522  This method contains the `CmdLineTask` interface to the ISR
1523  processing. All IO is handled here, freeing the `run()` method
1524  to manage only pixel-level calculations. The steps performed
1525  are:
1526  - Read in necessary detrending/isr/calibration data.
1527  - Process raw exposure in `run()`.
1528  - Persist the ISR-corrected exposure as "postISRCCD" if
1529  config.doWrite=True.
1530 
1531  Parameters
1532  ----------
1533  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1534  DataRef of the detector data to be processed
1535 
1536  Returns
1537  -------
1538  result : `lsst.pipe.base.Struct`
1539  Result struct with component:
1540  - ``exposure`` : `afw.image.Exposure`
1541  The fully ISR corrected exposure.
1542 
1543  Raises
1544  ------
1545  RuntimeError
1546  Raised if a configuration option is set to True, but the
1547  required calibration data does not exist.
1548 
1549  """
1550  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1551 
1552  ccdExposure = sensorRef.get(self.config.datasetType)
1553 
1554  camera = sensorRef.get("camera")
1555  isrData = self.readIsrData(sensorRef, ccdExposure)
1556 
1557  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1558 
1559  if self.config.doWrite:
1560  sensorRef.put(result.exposure, "postISRCCD")
1561  if result.preInterpolatedExposure is not None:
1562  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1563  if result.ossThumb is not None:
1564  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1565  if result.flattenedThumb is not None:
1566  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1567 
1568  return result
1569 
1570  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1571  """!Retrieve a calibration dataset for removing instrument signature.
1572 
1573  Parameters
1574  ----------
1575 
1576  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1577  DataRef of the detector data to find calibration datasets
1578  for.
1579  datasetType : `str`
1580  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1581  dateObs : `str`, optional
1582  Date of the observation. Used to correct butler failures
1583  when using fallback filters.
1584  immediate : `Bool`
1585  If True, disable butler proxies to enable error handling
1586  within this routine.
1587 
1588  Returns
1589  -------
1590  exposure : `lsst.afw.image.Exposure`
1591  Requested calibration frame.
1592 
1593  Raises
1594  ------
1595  RuntimeError
1596  Raised if no matching calibration frame can be found.
1597  """
1598  try:
1599  exp = dataRef.get(datasetType, immediate=immediate)
1600  except Exception as exc1:
1601  if not self.config.fallbackFilterName:
1602  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1603  try:
1604  if self.config.useFallbackDate and dateObs:
1605  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1606  dateObs=dateObs, immediate=immediate)
1607  else:
1608  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1609  except Exception as exc2:
1610  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1611  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1612  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1613 
1614  if self.config.doAssembleIsrExposures:
1615  exp = self.assembleCcd.assembleCcd(exp)
1616  return exp
1617 
1618  def ensureExposure(self, inputExp, camera, detectorNum):
1619  """Ensure that the data returned by Butler is a fully constructed exposure.
1620 
1621  ISR requires exposure-level image data for historical reasons, so if we did
1622  not recieve that from Butler, construct it from what we have, modifying the
1623  input in place.
1624 
1625  Parameters
1626  ----------
1627  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1628  `lsst.afw.image.ImageF`
1629  The input data structure obtained from Butler.
1630  camera : `lsst.afw.cameraGeom.camera`
1631  The camera associated with the image. Used to find the appropriate
1632  detector.
1633  detectorNum : `int`
1634  The detector this exposure should match.
1635 
1636  Returns
1637  -------
1638  inputExp : `lsst.afw.image.Exposure`
1639  The re-constructed exposure, with appropriate detector parameters.
1640 
1641  Raises
1642  ------
1643  TypeError
1644  Raised if the input data cannot be used to construct an exposure.
1645  """
1646  if isinstance(inputExp, afwImage.DecoratedImageU):
1647  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1648  elif isinstance(inputExp, afwImage.ImageF):
1649  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1650  elif isinstance(inputExp, afwImage.MaskedImageF):
1651  inputExp = afwImage.makeExposure(inputExp)
1652  elif isinstance(inputExp, afwImage.Exposure):
1653  pass
1654  elif inputExp is None:
1655  # Assume this will be caught by the setup if it is a problem.
1656  return inputExp
1657  else:
1658  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1659  (type(inputExp), ))
1660 
1661  if inputExp.getDetector() is None:
1662  inputExp.setDetector(camera[detectorNum])
1663 
1664  return inputExp
1665 
1666  def convertIntToFloat(self, exposure):
1667  """Convert exposure image from uint16 to float.
1668 
1669  If the exposure does not need to be converted, the input is
1670  immediately returned. For exposures that are converted to use
1671  floating point pixels, the variance is set to unity and the
1672  mask to zero.
1673 
1674  Parameters
1675  ----------
1676  exposure : `lsst.afw.image.Exposure`
1677  The raw exposure to be converted.
1678 
1679  Returns
1680  -------
1681  newexposure : `lsst.afw.image.Exposure`
1682  The input ``exposure``, converted to floating point pixels.
1683 
1684  Raises
1685  ------
1686  RuntimeError
1687  Raised if the exposure type cannot be converted to float.
1688 
1689  """
1690  if isinstance(exposure, afwImage.ExposureF):
1691  # Nothing to be done
1692  self.log.debug("Exposure already of type float.")
1693  return exposure
1694  if not hasattr(exposure, "convertF"):
1695  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1696 
1697  newexposure = exposure.convertF()
1698  newexposure.variance[:] = 1
1699  newexposure.mask[:] = 0x0
1700 
1701  return newexposure
1702 
1703  def maskAmplifier(self, ccdExposure, amp, defects):
1704  """Identify bad amplifiers, saturated and suspect pixels.
1705 
1706  Parameters
1707  ----------
1708  ccdExposure : `lsst.afw.image.Exposure`
1709  Input exposure to be masked.
1710  amp : `lsst.afw.table.AmpInfoCatalog`
1711  Catalog of parameters defining the amplifier on this
1712  exposure to mask.
1713  defects : `lsst.meas.algorithms.Defects`
1714  List of defects. Used to determine if the entire
1715  amplifier is bad.
1716 
1717  Returns
1718  -------
1719  badAmp : `Bool`
1720  If this is true, the entire amplifier area is covered by
1721  defects and unusable.
1722 
1723  """
1724  maskedImage = ccdExposure.getMaskedImage()
1725 
1726  badAmp = False
1727 
1728  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1729  # comparison with current defects definition.
1730  if defects is not None:
1731  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1732 
1733  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1734  # association with pixels in current ccdExposure).
1735  if badAmp:
1736  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1737  afwImage.PARENT)
1738  maskView = dataView.getMask()
1739  maskView |= maskView.getPlaneBitMask("BAD")
1740  del maskView
1741  return badAmp
1742 
1743  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1744  # Saturation and suspect pixels can be masked now, though.
1745  limits = dict()
1746  if self.config.doSaturation and not badAmp:
1747  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1748  if self.config.doSuspect and not badAmp:
1749  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1750  if math.isfinite(self.config.saturation):
1751  limits.update({self.config.saturatedMaskName: self.config.saturation})
1752 
1753  for maskName, maskThreshold in limits.items():
1754  if not math.isnan(maskThreshold):
1755  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1756  isrFunctions.makeThresholdMask(
1757  maskedImage=dataView,
1758  threshold=maskThreshold,
1759  growFootprints=0,
1760  maskName=maskName
1761  )
1762 
1763  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1764  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1765  afwImage.PARENT)
1766  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1767  self.config.suspectMaskName])
1768  if numpy.all(maskView.getArray() & maskVal > 0):
1769  badAmp = True
1770  maskView |= maskView.getPlaneBitMask("BAD")
1771 
1772  return badAmp
1773 
1774  def overscanCorrection(self, ccdExposure, amp):
1775  """Apply overscan correction in place.
1776 
1777  This method does initial pixel rejection of the overscan
1778  region. The overscan can also be optionally segmented to
1779  allow for discontinuous overscan responses to be fit
1780  separately. The actual overscan subtraction is performed by
1781  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1782  which is called here after the amplifier is preprocessed.
1783 
1784  Parameters
1785  ----------
1786  ccdExposure : `lsst.afw.image.Exposure`
1787  Exposure to have overscan correction performed.
1788  amp : `lsst.afw.table.AmpInfoCatalog`
1789  The amplifier to consider while correcting the overscan.
1790 
1791  Returns
1792  -------
1793  overscanResults : `lsst.pipe.base.Struct`
1794  Result struct with components:
1795  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1796  Value or fit subtracted from the amplifier image data.
1797  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1798  Value or fit subtracted from the overscan image data.
1799  - ``overscanImage`` : `lsst.afw.image.Image`
1800  Image of the overscan region with the overscan
1801  correction applied. This quantity is used to estimate
1802  the amplifier read noise empirically.
1803 
1804  Raises
1805  ------
1806  RuntimeError
1807  Raised if the ``amp`` does not contain raw pixel information.
1808 
1809  See Also
1810  --------
1811  lsst.ip.isr.isrFunctions.overscanCorrection
1812  """
1813  if not amp.getHasRawInfo():
1814  raise RuntimeError("This method must be executed on an amp with raw information.")
1815 
1816  if amp.getRawHorizontalOverscanBBox().isEmpty():
1817  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1818  return None
1819 
1820  statControl = afwMath.StatisticsControl()
1821  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1822 
1823  # Determine the bounding boxes
1824  dataBBox = amp.getRawDataBBox()
1825  oscanBBox = amp.getRawHorizontalOverscanBBox()
1826  dx0 = 0
1827  dx1 = 0
1828 
1829  prescanBBox = amp.getRawPrescanBBox()
1830  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1831  dx0 += self.config.overscanNumLeadingColumnsToSkip
1832  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1833  else:
1834  dx0 += self.config.overscanNumTrailingColumnsToSkip
1835  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1836 
1837  # Determine if we need to work on subregions of the amplifier and overscan.
1838  imageBBoxes = []
1839  overscanBBoxes = []
1840 
1841  if ((self.config.overscanBiasJump and
1842  self.config.overscanBiasJumpLocation) and
1843  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1844  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1845  self.config.overscanBiasJumpDevices)):
1846  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1847  yLower = self.config.overscanBiasJumpLocation
1848  yUpper = dataBBox.getHeight() - yLower
1849  else:
1850  yUpper = self.config.overscanBiasJumpLocation
1851  yLower = dataBBox.getHeight() - yUpper
1852 
1853  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1854  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1855  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1856  lsst.geom.Extent2I(dx0, 0),
1857  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1858  yLower)))
1859 
1860  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1861  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1862  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1863  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1864  yUpper)))
1865  else:
1866  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1867  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1868  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1869  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1870  oscanBBox.getHeight())))
1871 
1872  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1873  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1874  ampImage = ccdExposure.maskedImage[imageBBox]
1875  overscanImage = ccdExposure.maskedImage[overscanBBox]
1876 
1877  overscanArray = overscanImage.image.array
1878  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1879  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1880  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1881 
1882  statControl = afwMath.StatisticsControl()
1883  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1884 
1885  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1886  overscanImage=overscanImage,
1887  fitType=self.config.overscanFitType,
1888  order=self.config.overscanOrder,
1889  collapseRej=self.config.overscanNumSigmaClip,
1890  statControl=statControl,
1891  overscanIsInt=self.config.overscanIsInt
1892  )
1893 
1894  # Measure average overscan levels and record them in the metadata.
1895  levelStat = afwMath.MEDIAN
1896  sigmaStat = afwMath.STDEVCLIP
1897 
1898  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1899  self.config.qa.flatness.nIter)
1900  metadata = ccdExposure.getMetadata()
1901  ampNum = amp.getName()
1902  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1903  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1904  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1905  else:
1906  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1907  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1908  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1909 
1910  return overscanResults
1911 
1912  def updateVariance(self, ampExposure, amp, overscanImage=None):
1913  """Set the variance plane using the amplifier gain and read noise
1914 
1915  The read noise is calculated from the ``overscanImage`` if the
1916  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1917  the value from the amplifier data is used.
1918 
1919  Parameters
1920  ----------
1921  ampExposure : `lsst.afw.image.Exposure`
1922  Exposure to process.
1923  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1924  Amplifier detector data.
1925  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1926  Image of overscan, required only for empirical read noise.
1927 
1928  See also
1929  --------
1930  lsst.ip.isr.isrFunctions.updateVariance
1931  """
1932  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1933  gain = amp.getGain()
1934 
1935  if math.isnan(gain):
1936  gain = 1.0
1937  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1938  elif gain <= 0:
1939  patchedGain = 1.0
1940  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1941  amp.getName(), gain, patchedGain)
1942  gain = patchedGain
1943 
1944  if self.config.doEmpiricalReadNoise and overscanImage is None:
1945  self.log.info("Overscan is none for EmpiricalReadNoise.")
1946 
1947  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1948  stats = afwMath.StatisticsControl()
1949  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1950  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1951  self.log.info("Calculated empirical read noise for amp %s: %f.",
1952  amp.getName(), readNoise)
1953  else:
1954  readNoise = amp.getReadNoise()
1955 
1956  isrFunctions.updateVariance(
1957  maskedImage=ampExposure.getMaskedImage(),
1958  gain=gain,
1959  readNoise=readNoise,
1960  )
1961 
1962  def darkCorrection(self, exposure, darkExposure, invert=False):
1963  """!Apply dark correction in place.
1964 
1965  Parameters
1966  ----------
1967  exposure : `lsst.afw.image.Exposure`
1968  Exposure to process.
1969  darkExposure : `lsst.afw.image.Exposure`
1970  Dark exposure of the same size as ``exposure``.
1971  invert : `Bool`, optional
1972  If True, re-add the dark to an already corrected image.
1973 
1974  Raises
1975  ------
1976  RuntimeError
1977  Raised if either ``exposure`` or ``darkExposure`` do not
1978  have their dark time defined.
1979 
1980  See Also
1981  --------
1982  lsst.ip.isr.isrFunctions.darkCorrection
1983  """
1984  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1985  if math.isnan(expScale):
1986  raise RuntimeError("Exposure darktime is NAN.")
1987  if darkExposure.getInfo().getVisitInfo() is not None \
1988  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
1989  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1990  else:
1991  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1992  # so getDarkTime() does not exist.
1993  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1994  darkScale = 1.0
1995 
1996  isrFunctions.darkCorrection(
1997  maskedImage=exposure.getMaskedImage(),
1998  darkMaskedImage=darkExposure.getMaskedImage(),
1999  expScale=expScale,
2000  darkScale=darkScale,
2001  invert=invert,
2002  trimToFit=self.config.doTrimToMatchCalib
2003  )
2004 
2005  def doLinearize(self, detector):
2006  """!Check if linearization is needed for the detector cameraGeom.
2007 
2008  Checks config.doLinearize and the linearity type of the first
2009  amplifier.
2010 
2011  Parameters
2012  ----------
2013  detector : `lsst.afw.cameraGeom.Detector`
2014  Detector to get linearity type from.
2015 
2016  Returns
2017  -------
2018  doLinearize : `Bool`
2019  If True, linearization should be performed.
2020  """
2021  return self.config.doLinearize and \
2022  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2023 
2024  def flatCorrection(self, exposure, flatExposure, invert=False):
2025  """!Apply flat correction in place.
2026 
2027  Parameters
2028  ----------
2029  exposure : `lsst.afw.image.Exposure`
2030  Exposure to process.
2031  flatExposure : `lsst.afw.image.Exposure`
2032  Flat exposure of the same size as ``exposure``.
2033  invert : `Bool`, optional
2034  If True, unflatten an already flattened image.
2035 
2036  See Also
2037  --------
2038  lsst.ip.isr.isrFunctions.flatCorrection
2039  """
2040  isrFunctions.flatCorrection(
2041  maskedImage=exposure.getMaskedImage(),
2042  flatMaskedImage=flatExposure.getMaskedImage(),
2043  scalingType=self.config.flatScalingType,
2044  userScale=self.config.flatUserScale,
2045  invert=invert,
2046  trimToFit=self.config.doTrimToMatchCalib
2047  )
2048 
2049  def saturationDetection(self, exposure, amp):
2050  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2051 
2052  Parameters
2053  ----------
2054  exposure : `lsst.afw.image.Exposure`
2055  Exposure to process. Only the amplifier DataSec is processed.
2056  amp : `lsst.afw.table.AmpInfoCatalog`
2057  Amplifier detector data.
2058 
2059  See Also
2060  --------
2061  lsst.ip.isr.isrFunctions.makeThresholdMask
2062  """
2063  if not math.isnan(amp.getSaturation()):
2064  maskedImage = exposure.getMaskedImage()
2065  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2066  isrFunctions.makeThresholdMask(
2067  maskedImage=dataView,
2068  threshold=amp.getSaturation(),
2069  growFootprints=0,
2070  maskName=self.config.saturatedMaskName,
2071  )
2072 
2073  def saturationInterpolation(self, exposure):
2074  """!Interpolate over saturated pixels, in place.
2075 
2076  This method should be called after `saturationDetection`, to
2077  ensure that the saturated pixels have been identified in the
2078  SAT mask. It should also be called after `assembleCcd`, since
2079  saturated regions may cross amplifier boundaries.
2080 
2081  Parameters
2082  ----------
2083  exposure : `lsst.afw.image.Exposure`
2084  Exposure to process.
2085 
2086  See Also
2087  --------
2088  lsst.ip.isr.isrTask.saturationDetection
2089  lsst.ip.isr.isrFunctions.interpolateFromMask
2090  """
2091  isrFunctions.interpolateFromMask(
2092  maskedImage=exposure.getMaskedImage(),
2093  fwhm=self.config.fwhm,
2094  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2095  maskNameList=list(self.config.saturatedMaskName),
2096  )
2097 
2098  def suspectDetection(self, exposure, amp):
2099  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2100 
2101  Parameters
2102  ----------
2103  exposure : `lsst.afw.image.Exposure`
2104  Exposure to process. Only the amplifier DataSec is processed.
2105  amp : `lsst.afw.table.AmpInfoCatalog`
2106  Amplifier detector data.
2107 
2108  See Also
2109  --------
2110  lsst.ip.isr.isrFunctions.makeThresholdMask
2111 
2112  Notes
2113  -----
2114  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2115  This is intended to indicate pixels that may be affected by unknown systematics;
2116  for example if non-linearity corrections above a certain level are unstable
2117  then that would be a useful value for suspectLevel. A value of `nan` indicates
2118  that no such level exists and no pixels are to be masked as suspicious.
2119  """
2120  suspectLevel = amp.getSuspectLevel()
2121  if math.isnan(suspectLevel):
2122  return
2123 
2124  maskedImage = exposure.getMaskedImage()
2125  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2126  isrFunctions.makeThresholdMask(
2127  maskedImage=dataView,
2128  threshold=suspectLevel,
2129  growFootprints=0,
2130  maskName=self.config.suspectMaskName,
2131  )
2132 
2133  def maskDefect(self, exposure, defectBaseList):
2134  """!Mask defects using mask plane "BAD", in place.
2135 
2136  Parameters
2137  ----------
2138  exposure : `lsst.afw.image.Exposure`
2139  Exposure to process.
2140  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2141  `lsst.afw.image.DefectBase`.
2142  List of defects to mask.
2143 
2144  Notes
2145  -----
2146  Call this after CCD assembly, since defects may cross amplifier boundaries.
2147  """
2148  maskedImage = exposure.getMaskedImage()
2149  if not isinstance(defectBaseList, Defects):
2150  # Promotes DefectBase to Defect
2151  defectList = Defects(defectBaseList)
2152  else:
2153  defectList = defectBaseList
2154  defectList.maskPixels(maskedImage, maskName="BAD")
2155 
2156  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2157  """!Mask edge pixels with applicable mask plane.
2158 
2159  Parameters
2160  ----------
2161  exposure : `lsst.afw.image.Exposure`
2162  Exposure to process.
2163  numEdgePixels : `int`, optional
2164  Number of edge pixels to mask.
2165  maskPlane : `str`, optional
2166  Mask plane name to use.
2167  """
2168  maskedImage = exposure.getMaskedImage()
2169  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2170 
2171  if numEdgePixels > 0:
2172  goodBBox = maskedImage.getBBox()
2173  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2174  goodBBox.grow(-numEdgePixels)
2175  # Mask pixels outside goodBBox
2176  SourceDetectionTask.setEdgeBits(
2177  maskedImage,
2178  goodBBox,
2179  maskBitMask
2180  )
2181 
2182  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2183  """Mask and interpolate defects using mask plane "BAD", in place.
2184 
2185  Parameters
2186  ----------
2187  exposure : `lsst.afw.image.Exposure`
2188  Exposure to process.
2189  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2190  `lsst.afw.image.DefectBase`.
2191  List of defects to mask and interpolate.
2192 
2193  See Also
2194  --------
2195  lsst.ip.isr.isrTask.maskDefect()
2196  """
2197  self.maskDefect(exposure, defectBaseList)
2198  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2199  maskPlane="SUSPECT")
2200  isrFunctions.interpolateFromMask(
2201  maskedImage=exposure.getMaskedImage(),
2202  fwhm=self.config.fwhm,
2203  growSaturatedFootprints=0,
2204  maskNameList=["BAD"],
2205  )
2206 
2207  def maskNan(self, exposure):
2208  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2209 
2210  Parameters
2211  ----------
2212  exposure : `lsst.afw.image.Exposure`
2213  Exposure to process.
2214 
2215  Notes
2216  -----
2217  We mask over all NaNs, including those that are masked with
2218  other bits (because those may or may not be interpolated over
2219  later, and we want to remove all NaNs). Despite this
2220  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2221  the historical name.
2222  """
2223  maskedImage = exposure.getMaskedImage()
2224 
2225  # Find and mask NaNs
2226  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2227  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2228  numNans = maskNans(maskedImage, maskVal)
2229  self.metadata.set("NUMNANS", numNans)
2230  if numNans > 0:
2231  self.log.warn("There were %d unmasked NaNs.", numNans)
2232 
2233  def maskAndInterpolateNan(self, exposure):
2234  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2235 
2236  Parameters
2237  ----------
2238  exposure : `lsst.afw.image.Exposure`
2239  Exposure to process.
2240 
2241  See Also
2242  --------
2243  lsst.ip.isr.isrTask.maskNan()
2244  """
2245  self.maskNan(exposure)
2246  isrFunctions.interpolateFromMask(
2247  maskedImage=exposure.getMaskedImage(),
2248  fwhm=self.config.fwhm,
2249  growSaturatedFootprints=0,
2250  maskNameList=["UNMASKEDNAN"],
2251  )
2252 
2253  def measureBackground(self, exposure, IsrQaConfig=None):
2254  """Measure the image background in subgrids, for quality control purposes.
2255 
2256  Parameters
2257  ----------
2258  exposure : `lsst.afw.image.Exposure`
2259  Exposure to process.
2260  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2261  Configuration object containing parameters on which background
2262  statistics and subgrids to use.
2263  """
2264  if IsrQaConfig is not None:
2265  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2266  IsrQaConfig.flatness.nIter)
2267  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2268  statsControl.setAndMask(maskVal)
2269  maskedImage = exposure.getMaskedImage()
2270  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2271  skyLevel = stats.getValue(afwMath.MEDIAN)
2272  skySigma = stats.getValue(afwMath.STDEVCLIP)
2273  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2274  metadata = exposure.getMetadata()
2275  metadata.set('SKYLEVEL', skyLevel)
2276  metadata.set('SKYSIGMA', skySigma)
2277 
2278  # calcluating flatlevel over the subgrids
2279  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2280  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2281  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2282  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2283  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2284  skyLevels = numpy.zeros((nX, nY))
2285 
2286  for j in range(nY):
2287  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2288  for i in range(nX):
2289  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2290 
2291  xLLC = xc - meshXHalf
2292  yLLC = yc - meshYHalf
2293  xURC = xc + meshXHalf - 1
2294  yURC = yc + meshYHalf - 1
2295 
2296  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2297  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2298 
2299  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2300 
2301  good = numpy.where(numpy.isfinite(skyLevels))
2302  skyMedian = numpy.median(skyLevels[good])
2303  flatness = (skyLevels[good] - skyMedian) / skyMedian
2304  flatness_rms = numpy.std(flatness)
2305  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2306 
2307  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2308  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2309  nX, nY, flatness_pp, flatness_rms)
2310 
2311  metadata.set('FLATNESS_PP', float(flatness_pp))
2312  metadata.set('FLATNESS_RMS', float(flatness_rms))
2313  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2314  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2315  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2316 
2317  def roughZeroPoint(self, exposure):
2318  """Set an approximate magnitude zero point for the exposure.
2319 
2320  Parameters
2321  ----------
2322  exposure : `lsst.afw.image.Exposure`
2323  Exposure to process.
2324  """
2325  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2326  if filterName in self.config.fluxMag0T1:
2327  fluxMag0 = self.config.fluxMag0T1[filterName]
2328  else:
2329  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2330  fluxMag0 = self.config.defaultFluxMag0T1
2331 
2332  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2333  if not expTime > 0: # handle NaN as well as <= 0
2334  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2335  return
2336 
2337  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2338  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2339 
2340  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2341  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2342 
2343  Parameters
2344  ----------
2345  ccdExposure : `lsst.afw.image.Exposure`
2346  Exposure to process.
2347  fpPolygon : `lsst.afw.geom.Polygon`
2348  Polygon in focal plane coordinates.
2349  """
2350  # Get ccd corners in focal plane coordinates
2351  ccd = ccdExposure.getDetector()
2352  fpCorners = ccd.getCorners(FOCAL_PLANE)
2353  ccdPolygon = Polygon(fpCorners)
2354 
2355  # Get intersection of ccd corners with fpPolygon
2356  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2357 
2358  # Transform back to pixel positions and build new polygon
2359  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2360  validPolygon = Polygon(ccdPoints)
2361  ccdExposure.getInfo().setValidPolygon(validPolygon)
2362 
2363  @contextmanager
2364  def flatContext(self, exp, flat, dark=None):
2365  """Context manager that applies and removes flats and darks,
2366  if the task is configured to apply them.
2367 
2368  Parameters
2369  ----------
2370  exp : `lsst.afw.image.Exposure`
2371  Exposure to process.
2372  flat : `lsst.afw.image.Exposure`
2373  Flat exposure the same size as ``exp``.
2374  dark : `lsst.afw.image.Exposure`, optional
2375  Dark exposure the same size as ``exp``.
2376 
2377  Yields
2378  ------
2379  exp : `lsst.afw.image.Exposure`
2380  The flat and dark corrected exposure.
2381  """
2382  if self.config.doDark and dark is not None:
2383  self.darkCorrection(exp, dark)
2384  if self.config.doFlat:
2385  self.flatCorrection(exp, flat)
2386  try:
2387  yield exp
2388  finally:
2389  if self.config.doFlat:
2390  self.flatCorrection(exp, flat, invert=True)
2391  if self.config.doDark and dark is not None:
2392  self.darkCorrection(exp, dark, invert=True)
2393 
2394  def debugView(self, exposure, stepname):
2395  """Utility function to examine ISR exposure at different stages.
2396 
2397  Parameters
2398  ----------
2399  exposure : `lsst.afw.image.Exposure`
2400  Exposure to view.
2401  stepname : `str`
2402  State of processing to view.
2403  """
2404  frame = getDebugFrame(self._display, stepname)
2405  if frame:
2406  display = getDisplay(frame)
2407  display.scale('asinh', 'zscale')
2408  display.mtv(exposure)
2409  prompt = "Press Enter to continue [c]... "
2410  while True:
2411  ans = input(prompt).lower()
2412  if ans in ("", "c",):
2413  break
2414 
2415 
2416 class FakeAmp(object):
2417  """A Detector-like object that supports returning gain and saturation level
2418 
2419  This is used when the input exposure does not have a detector.
2420 
2421  Parameters
2422  ----------
2423  exposure : `lsst.afw.image.Exposure`
2424  Exposure to generate a fake amplifier for.
2425  config : `lsst.ip.isr.isrTaskConfig`
2426  Configuration to apply to the fake amplifier.
2427  """
2428 
2429  def __init__(self, exposure, config):
2430  self._bbox = exposure.getBBox(afwImage.LOCAL)
2432  self._gain = config.gain
2433  self._readNoise = config.readNoise
2434  self._saturation = config.saturation
2435 
2436  def getBBox(self):
2437  return self._bbox
2438 
2439  def getRawBBox(self):
2440  return self._bbox
2441 
2442  def getHasRawInfo(self):
2443  return True # but see getRawHorizontalOverscanBBox()
2444 
2446  return self._RawHorizontalOverscanBBox
2447 
2448  def getGain(self):
2449  return self._gain
2450 
2451  def getReadNoise(self):
2452  return self._readNoise
2453 
2454  def getSaturation(self):
2455  return self._saturation
2456 
2457  def getSuspectLevel(self):
2458  return float("NaN")
2459 
2460 
2461 class RunIsrConfig(pexConfig.Config):
2462  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2463 
2464 
2465 class RunIsrTask(pipeBase.CmdLineTask):
2466  """Task to wrap the default IsrTask to allow it to be retargeted.
2467 
2468  The standard IsrTask can be called directly from a command line
2469  program, but doing so removes the ability of the task to be
2470  retargeted. As most cameras override some set of the IsrTask
2471  methods, this would remove those data-specific methods in the
2472  output post-ISR images. This wrapping class fixes the issue,
2473  allowing identical post-ISR images to be generated by both the
2474  processCcd and isrTask code.
2475  """
2476  ConfigClass = RunIsrConfig
2477  _DefaultName = "runIsr"
2478 
2479  def __init__(self, *args, **kwargs):
2480  super().__init__(*args, **kwargs)
2481  self.makeSubtask("isr")
2482 
2483  def runDataRef(self, dataRef):
2484  """
2485  Parameters
2486  ----------
2487  dataRef : `lsst.daf.persistence.ButlerDataRef`
2488  data reference of the detector data to be processed
2489 
2490  Returns
2491  -------
2492  result : `pipeBase.Struct`
2493  Result struct with component:
2494 
2495  - exposure : `lsst.afw.image.Exposure`
2496  Post-ISR processed exposure.
2497  """
2498  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1519
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2253
def debugView(self, exposure, stepname)
Definition: isrTask.py:2394
def __init__(self, kwargs)
Definition: isrTask.py:810
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1618
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:893
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2156
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:819
def runDataRef(self, dataRef)
Definition: isrTask.py:2483
def __init__(self, args, kwargs)
Definition: isrTask.py:2479
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2317
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2182
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2445
def maskNan(self, exposure)
Definition: isrTask.py:2207
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2133
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1774
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1666
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:2024
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1063
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1962
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:2005
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2340
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1703
def __init__(self, config=None)
Definition: isrTask.py:179
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2364
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1570
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1912
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2233
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2098
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2073
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:2049
def __init__(self, exposure, config)
Definition: isrTask.py:2429