lsst.ip.isr  19.0.0-10-g17493b7+1
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.PrerequisiteInput(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ImageF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ImageF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="MaskedImageF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  newBFKernel = cT.PrerequisiteInput(
112  name='brighterFatterKernel',
113  doc="Newer complete kernel + gain solutions.",
114  storageClass="BrighterFatterKernel",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  defects = cT.PrerequisiteInput(
118  name='defects',
119  doc="Input defect tables.",
120  storageClass="DefectsList",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  opticsTransmission = cT.PrerequisiteInput(
124  name="transmission_optics",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the optics.",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = cT.PrerequisiteInput(
130  name="transmission_filter",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the filter.",
133  dimensions=["instrument", "physical_filter", "calibration_label"],
134  )
135  sensorTransmission = cT.PrerequisiteInput(
136  name="transmission_sensor",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the sensor.",
139  dimensions=["instrument", "calibration_label", "detector"],
140  )
141  atmosphereTransmission = cT.PrerequisiteInput(
142  name="transmission_atmosphere",
143  storageClass="TransmissionCurve",
144  doc="Transmission curve due to the atmosphere.",
145  dimensions=["instrument"],
146  )
147  illumMaskedImage = cT.PrerequisiteInput(
148  name="illum",
149  doc="Input illumination correction.",
150  storageClass="MaskedImageF",
151  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
152  )
153 
154  outputExposure = cT.Output(
155  name='postISRCCD',
156  doc="Output ISR processed exposure.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "visit", "detector"],
159  )
160  preInterpExposure = cT.Output(
161  name='preInterpISRCCD',
162  doc="Output ISR processed exposure, with pixels left uninterpolated.",
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = cT.Output(
167  name="OssThumb",
168  doc="Output Overscan-subtracted thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "visit", "detector"],
171  )
172  outputFlattenedThumbnail = cT.Output(
173  name="FlattenedThumb",
174  doc="Output flat-corrected thumbnail image.",
175  storageClass="Thumbnail",
176  dimensions=["instrument", "visit", "detector"],
177  )
178 
179  def __init__(self, *, config=None):
180  super().__init__(config=config)
181 
182  if config.doBias is not True:
183  self.prerequisiteInputs.discard("bias")
184  if config.doLinearize is not True:
185  self.prerequisiteInputs.discard("linearizer")
186  if config.doCrosstalk is not True:
187  self.prerequisiteInputs.discard("crosstalkSources")
188  if config.doBrighterFatter is not True:
189  self.prerequisiteInputs.discard("bfKernel")
190  self.prerequisiteInputs.discard("newBFKernel")
191  if config.doDefect is not True:
192  self.prerequisiteInputs.discard("defects")
193  if config.doDark is not True:
194  self.prerequisiteInputs.discard("dark")
195  if config.doFlat is not True:
196  self.prerequisiteInputs.discard("flat")
197  if config.doAttachTransmissionCurve is not True:
198  self.prerequisiteInputs.discard("opticsTransmission")
199  self.prerequisiteInputs.discard("filterTransmission")
200  self.prerequisiteInputs.discard("sensorTransmission")
201  self.prerequisiteInputs.discard("atmosphereTransmission")
202  if config.doUseOpticsTransmission is not True:
203  self.prerequisiteInputs.discard("opticsTransmission")
204  if config.doUseFilterTransmission is not True:
205  self.prerequisiteInputs.discard("filterTransmission")
206  if config.doUseSensorTransmission is not True:
207  self.prerequisiteInputs.discard("sensorTransmission")
208  if config.doUseAtmosphereTransmission is not True:
209  self.prerequisiteInputs.discard("atmosphereTransmission")
210  if config.doIlluminationCorrection is not True:
211  self.prerequisiteInputs.discard("illumMaskedImage")
212 
213  if config.doWrite is not True:
214  self.outputs.discard("outputExposure")
215  self.outputs.discard("preInterpExposure")
216  self.outputs.discard("outputFlattenedThumbnail")
217  self.outputs.discard("outputOssThumbnail")
218  if config.doSaveInterpPixels is not True:
219  self.outputs.discard("preInterpExposure")
220  if config.qa.doThumbnailOss is not True:
221  self.outputs.discard("outputOssThumbnail")
222  if config.qa.doThumbnailFlattened is not True:
223  self.outputs.discard("outputFlattenedThumbnail")
224 
225 
226 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
227  pipelineConnections=IsrTaskConnections):
228  """Configuration parameters for IsrTask.
229 
230  Items are grouped in the order in which they are executed by the task.
231  """
232  datasetType = pexConfig.Field(
233  dtype=str,
234  doc="Dataset type for input data; users will typically leave this alone, "
235  "but camera-specific ISR tasks will override it",
236  default="raw",
237  )
238 
239  fallbackFilterName = pexConfig.Field(
240  dtype=str,
241  doc="Fallback default filter name for calibrations.",
242  optional=True
243  )
244  useFallbackDate = pexConfig.Field(
245  dtype=bool,
246  doc="Pass observation date when using fallback filter.",
247  default=False,
248  )
249  expectWcs = pexConfig.Field(
250  dtype=bool,
251  default=True,
252  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
253  )
254  fwhm = pexConfig.Field(
255  dtype=float,
256  doc="FWHM of PSF in arcseconds.",
257  default=1.0,
258  )
259  qa = pexConfig.ConfigField(
260  dtype=isrQa.IsrQaConfig,
261  doc="QA related configuration options.",
262  )
263 
264  # Image conversion configuration
265  doConvertIntToFloat = pexConfig.Field(
266  dtype=bool,
267  doc="Convert integer raw images to floating point values?",
268  default=True,
269  )
270 
271  # Saturated pixel handling.
272  doSaturation = pexConfig.Field(
273  dtype=bool,
274  doc="Mask saturated pixels? NB: this is totally independent of the"
275  " interpolation option - this is ONLY setting the bits in the mask."
276  " To have them interpolated make sure doSaturationInterpolation=True",
277  default=True,
278  )
279  saturatedMaskName = pexConfig.Field(
280  dtype=str,
281  doc="Name of mask plane to use in saturation detection and interpolation",
282  default="SAT",
283  )
284  saturation = pexConfig.Field(
285  dtype=float,
286  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287  default=float("NaN"),
288  )
289  growSaturationFootprintSize = pexConfig.Field(
290  dtype=int,
291  doc="Number of pixels by which to grow the saturation footprints",
292  default=1,
293  )
294 
295  # Suspect pixel handling.
296  doSuspect = pexConfig.Field(
297  dtype=bool,
298  doc="Mask suspect pixels?",
299  default=False,
300  )
301  suspectMaskName = pexConfig.Field(
302  dtype=str,
303  doc="Name of mask plane to use for suspect pixels",
304  default="SUSPECT",
305  )
306  numEdgeSuspect = pexConfig.Field(
307  dtype=int,
308  doc="Number of edge pixels to be flagged as untrustworthy.",
309  default=0,
310  )
311 
312  # Initial masking options.
313  doSetBadRegions = pexConfig.Field(
314  dtype=bool,
315  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
316  default=True,
317  )
318  badStatistic = pexConfig.ChoiceField(
319  dtype=str,
320  doc="How to estimate the average value for BAD regions.",
321  default='MEANCLIP',
322  allowed={
323  "MEANCLIP": "Correct using the (clipped) mean of good data",
324  "MEDIAN": "Correct using the median of the good data",
325  },
326  )
327 
328  # Overscan subtraction configuration.
329  doOverscan = pexConfig.Field(
330  dtype=bool,
331  doc="Do overscan subtraction?",
332  default=True,
333  )
334  overscanFitType = pexConfig.ChoiceField(
335  dtype=str,
336  doc="The method for fitting the overscan bias level.",
337  default='MEDIAN',
338  allowed={
339  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
340  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
341  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
342  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
343  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
344  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
345  "MEAN": "Correct using the mean of the overscan region",
346  "MEANCLIP": "Correct using a clipped mean of the overscan region",
347  "MEDIAN": "Correct using the median of the overscan region",
348  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
349  },
350  )
351  overscanOrder = pexConfig.Field(
352  dtype=int,
353  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
354  "or number of spline knots if overscan fit type is a spline."),
355  default=1,
356  )
357  overscanNumSigmaClip = pexConfig.Field(
358  dtype=float,
359  doc="Rejection threshold (sigma) for collapsing overscan before fit",
360  default=3.0,
361  )
362  overscanIsInt = pexConfig.Field(
363  dtype=bool,
364  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" +
365  " and overscan.FitType=MEDIAN_PER_ROW.",
366  default=True,
367  )
368  overscanNumLeadingColumnsToSkip = pexConfig.Field(
369  dtype=int,
370  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
371  default=0,
372  )
373  overscanNumTrailingColumnsToSkip = pexConfig.Field(
374  dtype=int,
375  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
376  default=0,
377  )
378  overscanMaxDev = pexConfig.Field(
379  dtype=float,
380  doc="Maximum deviation from the median for overscan",
381  default=1000.0, check=lambda x: x > 0
382  )
383  overscanBiasJump = pexConfig.Field(
384  dtype=bool,
385  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
386  default=False,
387  )
388  overscanBiasJumpKeyword = pexConfig.Field(
389  dtype=str,
390  doc="Header keyword containing information about devices.",
391  default="NO_SUCH_KEY",
392  )
393  overscanBiasJumpDevices = pexConfig.ListField(
394  dtype=str,
395  doc="List of devices that need piecewise overscan correction.",
396  default=(),
397  )
398  overscanBiasJumpLocation = pexConfig.Field(
399  dtype=int,
400  doc="Location of bias jump along y-axis.",
401  default=0,
402  )
403 
404  # Amplifier to CCD assembly configuration
405  doAssembleCcd = pexConfig.Field(
406  dtype=bool,
407  default=True,
408  doc="Assemble amp-level exposures into a ccd-level exposure?"
409  )
410  assembleCcd = pexConfig.ConfigurableField(
411  target=AssembleCcdTask,
412  doc="CCD assembly task",
413  )
414 
415  # General calibration configuration.
416  doAssembleIsrExposures = pexConfig.Field(
417  dtype=bool,
418  default=False,
419  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
420  )
421  doTrimToMatchCalib = pexConfig.Field(
422  dtype=bool,
423  default=False,
424  doc="Trim raw data to match calibration bounding boxes?"
425  )
426 
427  # Bias subtraction.
428  doBias = pexConfig.Field(
429  dtype=bool,
430  doc="Apply bias frame correction?",
431  default=True,
432  )
433  biasDataProductName = pexConfig.Field(
434  dtype=str,
435  doc="Name of the bias data product",
436  default="bias",
437  )
438 
439  # Variance construction
440  doVariance = pexConfig.Field(
441  dtype=bool,
442  doc="Calculate variance?",
443  default=True
444  )
445  gain = pexConfig.Field(
446  dtype=float,
447  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
448  default=float("NaN"),
449  )
450  readNoise = pexConfig.Field(
451  dtype=float,
452  doc="The read noise to use if no Detector is present in the Exposure",
453  default=0.0,
454  )
455  doEmpiricalReadNoise = pexConfig.Field(
456  dtype=bool,
457  default=False,
458  doc="Calculate empirical read noise instead of value from AmpInfo data?"
459  )
460 
461  # Linearization.
462  doLinearize = pexConfig.Field(
463  dtype=bool,
464  doc="Correct for nonlinearity of the detector's response?",
465  default=True,
466  )
467 
468  # Crosstalk.
469  doCrosstalk = pexConfig.Field(
470  dtype=bool,
471  doc="Apply intra-CCD crosstalk correction?",
472  default=False,
473  )
474  doCrosstalkBeforeAssemble = pexConfig.Field(
475  dtype=bool,
476  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
477  default=False,
478  )
479  crosstalk = pexConfig.ConfigurableField(
480  target=CrosstalkTask,
481  doc="Intra-CCD crosstalk correction",
482  )
483 
484  # Masking options.
485  doDefect = pexConfig.Field(
486  dtype=bool,
487  doc="Apply correction for CCD defects, e.g. hot pixels?",
488  default=True,
489  )
490  doNanMasking = pexConfig.Field(
491  dtype=bool,
492  doc="Mask NAN pixels?",
493  default=True,
494  )
495  doWidenSaturationTrails = pexConfig.Field(
496  dtype=bool,
497  doc="Widen bleed trails based on their width?",
498  default=True
499  )
500 
501  # Brighter-Fatter correction.
502  doBrighterFatter = pexConfig.Field(
503  dtype=bool,
504  default=False,
505  doc="Apply the brighter fatter correction"
506  )
507  brighterFatterLevel = pexConfig.ChoiceField(
508  dtype=str,
509  default="DETECTOR",
510  doc="The level at which to correct for brighter-fatter.",
511  allowed={
512  "AMP": "Every amplifier treated separately.",
513  "DETECTOR": "One kernel per detector",
514  }
515  )
516  brighterFatterMaxIter = pexConfig.Field(
517  dtype=int,
518  default=10,
519  doc="Maximum number of iterations for the brighter fatter correction"
520  )
521  brighterFatterThreshold = pexConfig.Field(
522  dtype=float,
523  default=1000,
524  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
525  " absolute value of the difference between the current corrected image and the one"
526  " from the previous iteration summed over all the pixels."
527  )
528  brighterFatterApplyGain = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Should the gain be applied when applying the brighter fatter correction?"
532  )
533  brighterFatterMaskGrowSize = pexConfig.Field(
534  dtype=int,
535  default=0,
536  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
537  " when brighter-fatter correction is applied."
538  )
539 
540  # Dark subtraction.
541  doDark = pexConfig.Field(
542  dtype=bool,
543  doc="Apply dark frame correction?",
544  default=True,
545  )
546  darkDataProductName = pexConfig.Field(
547  dtype=str,
548  doc="Name of the dark data product",
549  default="dark",
550  )
551 
552  # Camera-specific stray light removal.
553  doStrayLight = pexConfig.Field(
554  dtype=bool,
555  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
556  default=False,
557  )
558  strayLight = pexConfig.ConfigurableField(
559  target=StrayLightTask,
560  doc="y-band stray light correction"
561  )
562 
563  # Flat correction.
564  doFlat = pexConfig.Field(
565  dtype=bool,
566  doc="Apply flat field correction?",
567  default=True,
568  )
569  flatDataProductName = pexConfig.Field(
570  dtype=str,
571  doc="Name of the flat data product",
572  default="flat",
573  )
574  flatScalingType = pexConfig.ChoiceField(
575  dtype=str,
576  doc="The method for scaling the flat on the fly.",
577  default='USER',
578  allowed={
579  "USER": "Scale by flatUserScale",
580  "MEAN": "Scale by the inverse of the mean",
581  "MEDIAN": "Scale by the inverse of the median",
582  },
583  )
584  flatUserScale = pexConfig.Field(
585  dtype=float,
586  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
587  default=1.0,
588  )
589  doTweakFlat = pexConfig.Field(
590  dtype=bool,
591  doc="Tweak flats to match observed amplifier ratios?",
592  default=False
593  )
594 
595  # Amplifier normalization based on gains instead of using flats configuration.
596  doApplyGains = pexConfig.Field(
597  dtype=bool,
598  doc="Correct the amplifiers for their gains instead of applying flat correction",
599  default=False,
600  )
601  normalizeGains = pexConfig.Field(
602  dtype=bool,
603  doc="Normalize all the amplifiers in each CCD to have the same median value.",
604  default=False,
605  )
606 
607  # Fringe correction.
608  doFringe = pexConfig.Field(
609  dtype=bool,
610  doc="Apply fringe correction?",
611  default=True,
612  )
613  fringe = pexConfig.ConfigurableField(
614  target=FringeTask,
615  doc="Fringe subtraction task",
616  )
617  fringeAfterFlat = pexConfig.Field(
618  dtype=bool,
619  doc="Do fringe subtraction after flat-fielding?",
620  default=True,
621  )
622 
623  # Distortion model application.
624  doAddDistortionModel = pexConfig.Field(
625  dtype=bool,
626  doc="Apply a distortion model based on camera geometry to the WCS?",
627  default=True,
628  deprecated=("Camera geometry is incorporated when reading the raw files."
629  " This option no longer is used, and will be removed after v19.")
630  )
631 
632  # Initial CCD-level background statistics options.
633  doMeasureBackground = pexConfig.Field(
634  dtype=bool,
635  doc="Measure the background level on the reduced image?",
636  default=False,
637  )
638 
639  # Camera-specific masking configuration.
640  doCameraSpecificMasking = pexConfig.Field(
641  dtype=bool,
642  doc="Mask camera-specific bad regions?",
643  default=False,
644  )
645  masking = pexConfig.ConfigurableField(
646  target=MaskingTask,
647  doc="Masking task."
648  )
649 
650  # Interpolation options.
651 
652  doInterpolate = pexConfig.Field(
653  dtype=bool,
654  doc="Interpolate masked pixels?",
655  default=True,
656  )
657  doSaturationInterpolation = pexConfig.Field(
658  dtype=bool,
659  doc="Perform interpolation over pixels masked as saturated?"
660  " NB: This is independent of doSaturation; if that is False this plane"
661  " will likely be blank, resulting in a no-op here.",
662  default=True,
663  )
664  doNanInterpolation = pexConfig.Field(
665  dtype=bool,
666  doc="Perform interpolation over pixels masked as NaN?"
667  " NB: This is independent of doNanMasking; if that is False this plane"
668  " will likely be blank, resulting in a no-op here.",
669  default=True,
670  )
671  doNanInterpAfterFlat = pexConfig.Field(
672  dtype=bool,
673  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
674  "also have to interpolate them before flat-fielding."),
675  default=False,
676  )
677  maskListToInterpolate = pexConfig.ListField(
678  dtype=str,
679  doc="List of mask planes that should be interpolated.",
680  default=['SAT', 'BAD', 'UNMASKEDNAN'],
681  )
682  doSaveInterpPixels = pexConfig.Field(
683  dtype=bool,
684  doc="Save a copy of the pre-interpolated pixel values?",
685  default=False,
686  )
687 
688  # Default photometric calibration options.
689  fluxMag0T1 = pexConfig.DictField(
690  keytype=str,
691  itemtype=float,
692  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
693  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
694  ))
695  )
696  defaultFluxMag0T1 = pexConfig.Field(
697  dtype=float,
698  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
699  default=pow(10.0, 0.4*28.0)
700  )
701 
702  # Vignette correction configuration.
703  doVignette = pexConfig.Field(
704  dtype=bool,
705  doc="Apply vignetting parameters?",
706  default=False,
707  )
708  vignette = pexConfig.ConfigurableField(
709  target=VignetteTask,
710  doc="Vignetting task.",
711  )
712 
713  # Transmission curve configuration.
714  doAttachTransmissionCurve = pexConfig.Field(
715  dtype=bool,
716  default=False,
717  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
718  )
719  doUseOpticsTransmission = pexConfig.Field(
720  dtype=bool,
721  default=True,
722  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
723  )
724  doUseFilterTransmission = pexConfig.Field(
725  dtype=bool,
726  default=True,
727  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
728  )
729  doUseSensorTransmission = pexConfig.Field(
730  dtype=bool,
731  default=True,
732  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
733  )
734  doUseAtmosphereTransmission = pexConfig.Field(
735  dtype=bool,
736  default=True,
737  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
738  )
739 
740  # Illumination correction.
741  doIlluminationCorrection = pexConfig.Field(
742  dtype=bool,
743  default=False,
744  doc="Perform illumination correction?"
745  )
746  illuminationCorrectionDataProductName = pexConfig.Field(
747  dtype=str,
748  doc="Name of the illumination correction data product.",
749  default="illumcor",
750  )
751  illumScale = pexConfig.Field(
752  dtype=float,
753  doc="Scale factor for the illumination correction.",
754  default=1.0,
755  )
756  illumFilters = pexConfig.ListField(
757  dtype=str,
758  default=[],
759  doc="Only perform illumination correction for these filters."
760  )
761 
762  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
763  doWrite = pexConfig.Field(
764  dtype=bool,
765  doc="Persist postISRCCD?",
766  default=True,
767  )
768 
769  def validate(self):
770  super().validate()
771  if self.doFlat and self.doApplyGains:
772  raise ValueError("You may not specify both doFlat and doApplyGains")
773  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
774  self.config.maskListToInterpolate.append("SAT")
775  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
776  self.config.maskListToInterpolate.append("UNMASKEDNAN")
777 
778 
779 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
780  """Apply common instrument signature correction algorithms to a raw frame.
781 
782  The process for correcting imaging data is very similar from
783  camera to camera. This task provides a vanilla implementation of
784  doing these corrections, including the ability to turn certain
785  corrections off if they are not needed. The inputs to the primary
786  method, `run()`, are a raw exposure to be corrected and the
787  calibration data products. The raw input is a single chip sized
788  mosaic of all amps including overscans and other non-science
789  pixels. The method `runDataRef()` identifies and defines the
790  calibration data products, and is intended for use by a
791  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
792  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
793  subclassed for different camera, although the most camera specific
794  methods have been split into subtasks that can be redirected
795  appropriately.
796 
797  The __init__ method sets up the subtasks for ISR processing, using
798  the defaults from `lsst.ip.isr`.
799 
800  Parameters
801  ----------
802  args : `list`
803  Positional arguments passed to the Task constructor. None used at this time.
804  kwargs : `dict`, optional
805  Keyword arguments passed on to the Task constructor. None used at this time.
806  """
807  ConfigClass = IsrTaskConfig
808  _DefaultName = "isr"
809 
810  def __init__(self, **kwargs):
811  super().__init__(**kwargs)
812  self.makeSubtask("assembleCcd")
813  self.makeSubtask("crosstalk")
814  self.makeSubtask("strayLight")
815  self.makeSubtask("fringe")
816  self.makeSubtask("masking")
817  self.makeSubtask("vignette")
818 
819  def runQuantum(self, butlerQC, inputRefs, outputRefs):
820  inputs = butlerQC.get(inputRefs)
821 
822  try:
823  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
824  except Exception as e:
825  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
826  (inputRefs, e))
827 
828  inputs['isGen3'] = True
829 
830  detector = inputs['ccdExposure'].getDetector()
831  if self.doLinearize(detector) is True:
832  if 'linearizer' not in inputs:
833  linearityName = detector.getAmplifiers()[0].getLinearityType()
834  inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
835 
836  if self.config.doDefect is True:
837  if "defects" in inputs and inputs['defects'] is not None:
838  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
839  # masking expects a list of defects defined by their bounding box
840  if not isinstance(inputs["defects"], Defects):
841  inputs["defects"] = Defects.fromTable(inputs["defects"])
842 
843  # Load the correct style of brighter fatter kernel, and repack
844  # the information as a numpy array.
845  if self.config.doBrighterFatter:
846  brighterFatterKernel = inputs.pop('newBFKernel', None)
847  if brighterFatterKernel is None:
848  brighterFatterKernel = inputs.get('bfKernel', None)
849 
850  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
851  detId = detector.getId()
852  inputs['bfGains'] = brighterFatterKernel.gain
853  # If the kernel is not an ndarray, it's the cp_pipe version
854  # so extract the kernel for this detector, or raise an error
855  if self.config.brighterFatterLevel == 'DETECTOR':
856  if brighterFatterKernel.detectorKernel:
857  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
858  elif brighterFatterKernel.detectorKernelFromAmpKernels:
859  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
860  else:
861  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
862  else:
863  # TODO DM-15631 for implementing this
864  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
865 
866  # Broken: DM-17169
867  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
868  # fixed for non-HSC cameras in the future.
869  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
870  # if self.config.doCrosstalk else None)
871 
872  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
873  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
874  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
875  expId=expId,
876  assembler=self.assembleCcd
877  if self.config.doAssembleIsrExposures else None)
878  else:
879  inputs['fringes'] = pipeBase.Struct(fringes=None)
880 
881  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
882  if 'strayLightData' not in inputs:
883  inputs['strayLightData'] = None
884 
885  outputs = self.run(**inputs)
886  butlerQC.put(outputs, outputRefs)
887 
888  def readIsrData(self, dataRef, rawExposure):
889  """!Retrieve necessary frames for instrument signature removal.
890 
891  Pre-fetching all required ISR data products limits the IO
892  required by the ISR. Any conflict between the calibration data
893  available and that needed for ISR is also detected prior to
894  doing processing, allowing it to fail quickly.
895 
896  Parameters
897  ----------
898  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
899  Butler reference of the detector data to be processed
900  rawExposure : `afw.image.Exposure`
901  The raw exposure that will later be corrected with the
902  retrieved calibration data; should not be modified in this
903  method.
904 
905  Returns
906  -------
907  result : `lsst.pipe.base.Struct`
908  Result struct with components (which may be `None`):
909  - ``bias``: bias calibration frame (`afw.image.Exposure`)
910  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
911  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
912  - ``dark``: dark calibration frame (`afw.image.Exposure`)
913  - ``flat``: flat calibration frame (`afw.image.Exposure`)
914  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
915  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
916  - ``fringes``: `lsst.pipe.base.Struct` with components:
917  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
918  - ``seed``: random seed derived from the ccdExposureId for random
919  number generator (`uint32`).
920  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
921  A ``TransmissionCurve`` that represents the throughput of the optics,
922  to be evaluated in focal-plane coordinates.
923  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
924  A ``TransmissionCurve`` that represents the throughput of the filter
925  itself, to be evaluated in focal-plane coordinates.
926  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
927  A ``TransmissionCurve`` that represents the throughput of the sensor
928  itself, to be evaluated in post-assembly trimmed detector coordinates.
929  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
930  A ``TransmissionCurve`` that represents the throughput of the
931  atmosphere, assumed to be spatially constant.
932  - ``strayLightData`` : `object`
933  An opaque object containing calibration information for
934  stray-light correction. If `None`, no correction will be
935  performed.
936  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
937 
938  Raises
939  ------
940  NotImplementedError :
941  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
942  """
943  try:
944  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
945  dateObs = dateObs.toPython().isoformat()
946  except RuntimeError:
947  self.log.warn("Unable to identify dateObs for rawExposure.")
948  dateObs = None
949 
950  ccd = rawExposure.getDetector()
951  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
952  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
953  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
954  if self.config.doBias else None)
955  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
956  linearizer = (dataRef.get("linearizer", immediate=True)
957  if self.doLinearize(ccd) else None)
958  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
959  if self.config.doCrosstalk else None)
960  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
961  if self.config.doDark else None)
962  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
963  dateObs=dateObs)
964  if self.config.doFlat else None)
965 
966  brighterFatterKernel = None
967  brighterFatterGains = None
968  if self.config.doBrighterFatter is True:
969  try:
970  # Use the new-style cp_pipe version of the kernel if it exists
971  # If using a new-style kernel, always use the self-consistent
972  # gains, i.e. the ones inside the kernel object itself
973  brighterFatterKernel = dataRef.get("brighterFatterKernel")
974  brighterFatterGains = brighterFatterKernel.gain
975  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
976  except NoResults:
977  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
978  brighterFatterKernel = dataRef.get("bfKernel")
979  self.log.info("Old style bright-fatter kernel (np.array) loaded")
980  except NoResults:
981  brighterFatterKernel = None
982  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
983  # If the kernel is not an ndarray, it's the cp_pipe version
984  # so extract the kernel for this detector, or raise an error
985  if self.config.brighterFatterLevel == 'DETECTOR':
986  if brighterFatterKernel.detectorKernel:
987  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
988  elif brighterFatterKernel.detectorKernelFromAmpKernels:
989  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
990  else:
991  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
992  else:
993  # TODO DM-15631 for implementing this
994  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
995 
996  defectList = (dataRef.get("defects")
997  if self.config.doDefect else None)
998  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
999  if self.config.doAssembleIsrExposures else None)
1000  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1001  else pipeBase.Struct(fringes=None))
1002 
1003  if self.config.doAttachTransmissionCurve:
1004  opticsTransmission = (dataRef.get("transmission_optics")
1005  if self.config.doUseOpticsTransmission else None)
1006  filterTransmission = (dataRef.get("transmission_filter")
1007  if self.config.doUseFilterTransmission else None)
1008  sensorTransmission = (dataRef.get("transmission_sensor")
1009  if self.config.doUseSensorTransmission else None)
1010  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1011  if self.config.doUseAtmosphereTransmission else None)
1012  else:
1013  opticsTransmission = None
1014  filterTransmission = None
1015  sensorTransmission = None
1016  atmosphereTransmission = None
1017 
1018  if self.config.doStrayLight:
1019  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1020  else:
1021  strayLightData = None
1022 
1023  illumMaskedImage = (self.getIsrExposure(dataRef,
1024  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1025  if (self.config.doIlluminationCorrection and
1026  filterName in self.config.illumFilters)
1027  else None)
1028 
1029  # Struct should include only kwargs to run()
1030  return pipeBase.Struct(bias=biasExposure,
1031  linearizer=linearizer,
1032  crosstalkSources=crosstalkSources,
1033  dark=darkExposure,
1034  flat=flatExposure,
1035  bfKernel=brighterFatterKernel,
1036  bfGains=brighterFatterGains,
1037  defects=defectList,
1038  fringes=fringeStruct,
1039  opticsTransmission=opticsTransmission,
1040  filterTransmission=filterTransmission,
1041  sensorTransmission=sensorTransmission,
1042  atmosphereTransmission=atmosphereTransmission,
1043  strayLightData=strayLightData,
1044  illumMaskedImage=illumMaskedImage
1045  )
1046 
1047  @pipeBase.timeMethod
1048  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1049  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1050  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1051  sensorTransmission=None, atmosphereTransmission=None,
1052  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1053  isGen3=False,
1054  ):
1055  """!Perform instrument signature removal on an exposure.
1056 
1057  Steps included in the ISR processing, in order performed, are:
1058  - saturation and suspect pixel masking
1059  - overscan subtraction
1060  - CCD assembly of individual amplifiers
1061  - bias subtraction
1062  - variance image construction
1063  - linearization of non-linear response
1064  - crosstalk masking
1065  - brighter-fatter correction
1066  - dark subtraction
1067  - fringe correction
1068  - stray light subtraction
1069  - flat correction
1070  - masking of known defects and camera specific features
1071  - vignette calculation
1072  - appending transmission curve and distortion model
1073 
1074  Parameters
1075  ----------
1076  ccdExposure : `lsst.afw.image.Exposure`
1077  The raw exposure that is to be run through ISR. The
1078  exposure is modified by this method.
1079  camera : `lsst.afw.cameraGeom.Camera`, optional
1080  The camera geometry for this exposure. Used to select the
1081  distortion model appropriate for this data.
1082  bias : `lsst.afw.image.Exposure`, optional
1083  Bias calibration frame.
1084  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1085  Functor for linearization.
1086  crosstalkSources : `list`, optional
1087  List of possible crosstalk sources.
1088  dark : `lsst.afw.image.Exposure`, optional
1089  Dark calibration frame.
1090  flat : `lsst.afw.image.Exposure`, optional
1091  Flat calibration frame.
1092  bfKernel : `numpy.ndarray`, optional
1093  Brighter-fatter kernel.
1094  bfGains : `dict` of `float`, optional
1095  Gains used to override the detector's nominal gains for the
1096  brighter-fatter correction. A dict keyed by amplifier name for
1097  the detector in question.
1098  defects : `lsst.meas.algorithms.Defects`, optional
1099  List of defects.
1100  fringes : `lsst.pipe.base.Struct`, optional
1101  Struct containing the fringe correction data, with
1102  elements:
1103  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1104  - ``seed``: random seed derived from the ccdExposureId for random
1105  number generator (`uint32`)
1106  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1107  A ``TransmissionCurve`` that represents the throughput of the optics,
1108  to be evaluated in focal-plane coordinates.
1109  filterTransmission : `lsst.afw.image.TransmissionCurve`
1110  A ``TransmissionCurve`` that represents the throughput of the filter
1111  itself, to be evaluated in focal-plane coordinates.
1112  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1113  A ``TransmissionCurve`` that represents the throughput of the sensor
1114  itself, to be evaluated in post-assembly trimmed detector coordinates.
1115  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1116  A ``TransmissionCurve`` that represents the throughput of the
1117  atmosphere, assumed to be spatially constant.
1118  detectorNum : `int`, optional
1119  The integer number for the detector to process.
1120  isGen3 : bool, optional
1121  Flag this call to run() as using the Gen3 butler environment.
1122  strayLightData : `object`, optional
1123  Opaque object containing calibration information for stray-light
1124  correction. If `None`, no correction will be performed.
1125  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1126  Illumination correction image.
1127 
1128  Returns
1129  -------
1130  result : `lsst.pipe.base.Struct`
1131  Result struct with component:
1132  - ``exposure`` : `afw.image.Exposure`
1133  The fully ISR corrected exposure.
1134  - ``outputExposure`` : `afw.image.Exposure`
1135  An alias for `exposure`
1136  - ``ossThumb`` : `numpy.ndarray`
1137  Thumbnail image of the exposure after overscan subtraction.
1138  - ``flattenedThumb`` : `numpy.ndarray`
1139  Thumbnail image of the exposure after flat-field correction.
1140 
1141  Raises
1142  ------
1143  RuntimeError
1144  Raised if a configuration option is set to True, but the
1145  required calibration data has not been specified.
1146 
1147  Notes
1148  -----
1149  The current processed exposure can be viewed by setting the
1150  appropriate lsstDebug entries in the `debug.display`
1151  dictionary. The names of these entries correspond to some of
1152  the IsrTaskConfig Boolean options, with the value denoting the
1153  frame to use. The exposure is shown inside the matching
1154  option check and after the processing of that step has
1155  finished. The steps with debug points are:
1156 
1157  doAssembleCcd
1158  doBias
1159  doCrosstalk
1160  doBrighterFatter
1161  doDark
1162  doFringe
1163  doStrayLight
1164  doFlat
1165 
1166  In addition, setting the "postISRCCD" entry displays the
1167  exposure after all ISR processing has finished.
1168 
1169  """
1170 
1171  if isGen3 is True:
1172  # Gen3 currently cannot automatically do configuration overrides.
1173  # DM-15257 looks to discuss this issue.
1174  # Configure input exposures;
1175  if detectorNum is None:
1176  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1177 
1178  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1179  bias = self.ensureExposure(bias, camera, detectorNum)
1180  dark = self.ensureExposure(dark, camera, detectorNum)
1181  flat = self.ensureExposure(flat, camera, detectorNum)
1182  else:
1183  if isinstance(ccdExposure, ButlerDataRef):
1184  return self.runDataRef(ccdExposure)
1185 
1186  ccd = ccdExposure.getDetector()
1187  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1188 
1189  if not ccd:
1190  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1191  ccd = [FakeAmp(ccdExposure, self.config)]
1192 
1193  # Validate Input
1194  if self.config.doBias and bias is None:
1195  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1196  if self.doLinearize(ccd) and linearizer is None:
1197  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1198  if self.config.doBrighterFatter and bfKernel is None:
1199  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1200  if self.config.doDark and dark is None:
1201  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1202  if self.config.doFlat and flat is None:
1203  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1204  if self.config.doDefect and defects is None:
1205  raise RuntimeError("Must supply defects if config.doDefect=True.")
1206  if (self.config.doFringe and filterName in self.fringe.config.filters and
1207  fringes.fringes is None):
1208  # The `fringes` object needs to be a pipeBase.Struct, as
1209  # we use it as a `dict` for the parameters of
1210  # `FringeTask.run()`. The `fringes.fringes` `list` may
1211  # not be `None` if `doFringe=True`. Otherwise, raise.
1212  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1213  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1214  illumMaskedImage is None):
1215  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1216 
1217  # Begin ISR processing.
1218  if self.config.doConvertIntToFloat:
1219  self.log.info("Converting exposure to floating point values.")
1220  ccdExposure = self.convertIntToFloat(ccdExposure)
1221 
1222  # Amplifier level processing.
1223  overscans = []
1224  for amp in ccd:
1225  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1226  if ccdExposure.getBBox().contains(amp.getBBox()):
1227  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1228  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1229 
1230  if self.config.doOverscan and not badAmp:
1231  # Overscan correction on amp-by-amp basis.
1232  overscanResults = self.overscanCorrection(ccdExposure, amp)
1233  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1234  if overscanResults is not None and \
1235  self.config.qa is not None and self.config.qa.saveStats is True:
1236  if isinstance(overscanResults.overscanFit, float):
1237  qaMedian = overscanResults.overscanFit
1238  qaStdev = float("NaN")
1239  else:
1240  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1241  afwMath.MEDIAN | afwMath.STDEVCLIP)
1242  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1243  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1244 
1245  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1246  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1247  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1248  amp.getName(), qaMedian, qaStdev)
1249  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1250  else:
1251  if badAmp:
1252  self.log.warn("Amplifier %s is bad.", amp.getName())
1253  overscanResults = None
1254 
1255  overscans.append(overscanResults if overscanResults is not None else None)
1256  else:
1257  self.log.info("Skipped OSCAN for %s.", amp.getName())
1258 
1259  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1260  self.log.info("Applying crosstalk correction.")
1261  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1262  self.debugView(ccdExposure, "doCrosstalk")
1263 
1264  if self.config.doAssembleCcd:
1265  self.log.info("Assembling CCD from amplifiers.")
1266  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1267 
1268  if self.config.expectWcs and not ccdExposure.getWcs():
1269  self.log.warn("No WCS found in input exposure.")
1270  self.debugView(ccdExposure, "doAssembleCcd")
1271 
1272  ossThumb = None
1273  if self.config.qa.doThumbnailOss:
1274  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1275 
1276  if self.config.doBias:
1277  self.log.info("Applying bias correction.")
1278  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1279  trimToFit=self.config.doTrimToMatchCalib)
1280  self.debugView(ccdExposure, "doBias")
1281 
1282  if self.config.doVariance:
1283  for amp, overscanResults in zip(ccd, overscans):
1284  if ccdExposure.getBBox().contains(amp.getBBox()):
1285  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1286  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1287  if overscanResults is not None:
1288  self.updateVariance(ampExposure, amp,
1289  overscanImage=overscanResults.overscanImage)
1290  else:
1291  self.updateVariance(ampExposure, amp,
1292  overscanImage=None)
1293  if self.config.qa is not None and self.config.qa.saveStats is True:
1294  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1295  afwMath.MEDIAN | afwMath.STDEVCLIP)
1296  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1297  qaStats.getValue(afwMath.MEDIAN))
1298  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1299  qaStats.getValue(afwMath.STDEVCLIP))
1300  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1301  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1302  qaStats.getValue(afwMath.STDEVCLIP))
1303 
1304  if self.doLinearize(ccd):
1305  self.log.info("Applying linearizer.")
1306  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1307 
1308  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1309  self.log.info("Applying crosstalk correction.")
1310  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1311  self.debugView(ccdExposure, "doCrosstalk")
1312 
1313  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1314  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1315  if self.config.doDefect:
1316  self.log.info("Masking defects.")
1317  self.maskDefect(ccdExposure, defects)
1318 
1319  if self.config.numEdgeSuspect > 0:
1320  self.log.info("Masking edges as SUSPECT.")
1321  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1322  maskPlane="SUSPECT")
1323 
1324  if self.config.doNanMasking:
1325  self.log.info("Masking NAN value pixels.")
1326  self.maskNan(ccdExposure)
1327 
1328  if self.config.doWidenSaturationTrails:
1329  self.log.info("Widening saturation trails.")
1330  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1331 
1332  if self.config.doCameraSpecificMasking:
1333  self.log.info("Masking regions for camera specific reasons.")
1334  self.masking.run(ccdExposure)
1335 
1336  if self.config.doBrighterFatter:
1337  # We need to apply flats and darks before we can interpolate, and we
1338  # need to interpolate before we do B-F, but we do B-F without the
1339  # flats and darks applied so we can work in units of electrons or holes.
1340  # This context manager applies and then removes the darks and flats.
1341  #
1342  # We also do not want to interpolate values here, so operate on temporary
1343  # images so we can apply only the BF-correction and roll back the
1344  # interpolation.
1345  interpExp = ccdExposure.clone()
1346  with self.flatContext(interpExp, flat, dark):
1347  isrFunctions.interpolateFromMask(
1348  maskedImage=interpExp.getMaskedImage(),
1349  fwhm=self.config.fwhm,
1350  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1351  maskNameList=self.config.maskListToInterpolate
1352  )
1353  bfExp = interpExp.clone()
1354 
1355  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1356  type(bfKernel), type(bfGains))
1357  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1358  self.config.brighterFatterMaxIter,
1359  self.config.brighterFatterThreshold,
1360  self.config.brighterFatterApplyGain,
1361  bfGains)
1362  if bfResults[1] == self.config.brighterFatterMaxIter:
1363  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1364  bfResults[0])
1365  else:
1366  self.log.info("Finished brighter fatter correction in %d iterations.",
1367  bfResults[1])
1368  image = ccdExposure.getMaskedImage().getImage()
1369  bfCorr = bfExp.getMaskedImage().getImage()
1370  bfCorr -= interpExp.getMaskedImage().getImage()
1371  image += bfCorr
1372 
1373  # Applying the brighter-fatter correction applies a
1374  # convolution to the science image. At the edges this
1375  # convolution may not have sufficient valid pixels to
1376  # produce a valid correction. Mark pixels within the size
1377  # of the brighter-fatter kernel as EDGE to warn of this
1378  # fact.
1379  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1380  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1381  maskPlane="EDGE")
1382 
1383  if self.config.brighterFatterMaskGrowSize > 0:
1384  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1385  for maskPlane in self.config.maskListToInterpolate:
1386  isrFunctions.growMasks(ccdExposure.getMask(),
1387  radius=self.config.brighterFatterMaskGrowSize,
1388  maskNameList=maskPlane,
1389  maskValue=maskPlane)
1390 
1391  self.debugView(ccdExposure, "doBrighterFatter")
1392 
1393  if self.config.doDark:
1394  self.log.info("Applying dark correction.")
1395  self.darkCorrection(ccdExposure, dark)
1396  self.debugView(ccdExposure, "doDark")
1397 
1398  if self.config.doFringe and not self.config.fringeAfterFlat:
1399  self.log.info("Applying fringe correction before flat.")
1400  self.fringe.run(ccdExposure, **fringes.getDict())
1401  self.debugView(ccdExposure, "doFringe")
1402 
1403  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1404  self.log.info("Checking strayLight correction.")
1405  self.strayLight.run(ccdExposure, strayLightData)
1406  self.debugView(ccdExposure, "doStrayLight")
1407 
1408  if self.config.doFlat:
1409  self.log.info("Applying flat correction.")
1410  self.flatCorrection(ccdExposure, flat)
1411  self.debugView(ccdExposure, "doFlat")
1412 
1413  if self.config.doApplyGains:
1414  self.log.info("Applying gain correction instead of flat.")
1415  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1416 
1417  if self.config.doFringe and self.config.fringeAfterFlat:
1418  self.log.info("Applying fringe correction after flat.")
1419  self.fringe.run(ccdExposure, **fringes.getDict())
1420 
1421  if self.config.doVignette:
1422  self.log.info("Constructing Vignette polygon.")
1423  self.vignettePolygon = self.vignette.run(ccdExposure)
1424 
1425  if self.config.vignette.doWriteVignettePolygon:
1426  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1427 
1428  if self.config.doAttachTransmissionCurve:
1429  self.log.info("Adding transmission curves.")
1430  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1431  filterTransmission=filterTransmission,
1432  sensorTransmission=sensorTransmission,
1433  atmosphereTransmission=atmosphereTransmission)
1434 
1435  flattenedThumb = None
1436  if self.config.qa.doThumbnailFlattened:
1437  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1438 
1439  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1440  self.log.info("Performing illumination correction.")
1441  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1442  illumMaskedImage, illumScale=self.config.illumScale,
1443  trimToFit=self.config.doTrimToMatchCalib)
1444 
1445  preInterpExp = None
1446  if self.config.doSaveInterpPixels:
1447  preInterpExp = ccdExposure.clone()
1448 
1449  # Reset and interpolate bad pixels.
1450  #
1451  # Large contiguous bad regions (which should have the BAD mask
1452  # bit set) should have their values set to the image median.
1453  # This group should include defects and bad amplifiers. As the
1454  # area covered by these defects are large, there's little
1455  # reason to expect that interpolation would provide a more
1456  # useful value.
1457  #
1458  # Smaller defects can be safely interpolated after the larger
1459  # regions have had their pixel values reset. This ensures
1460  # that the remaining defects adjacent to bad amplifiers (as an
1461  # example) do not attempt to interpolate extreme values.
1462  if self.config.doSetBadRegions:
1463  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1464  if badPixelCount > 0:
1465  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1466 
1467  if self.config.doInterpolate:
1468  self.log.info("Interpolating masked pixels.")
1469  isrFunctions.interpolateFromMask(
1470  maskedImage=ccdExposure.getMaskedImage(),
1471  fwhm=self.config.fwhm,
1472  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1473  maskNameList=list(self.config.maskListToInterpolate)
1474  )
1475 
1476  self.roughZeroPoint(ccdExposure)
1477 
1478  if self.config.doMeasureBackground:
1479  self.log.info("Measuring background level.")
1480  self.measureBackground(ccdExposure, self.config.qa)
1481 
1482  if self.config.qa is not None and self.config.qa.saveStats is True:
1483  for amp in ccd:
1484  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1485  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1486  afwMath.MEDIAN | afwMath.STDEVCLIP)
1487  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1488  qaStats.getValue(afwMath.MEDIAN))
1489  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1490  qaStats.getValue(afwMath.STDEVCLIP))
1491  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1492  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1493  qaStats.getValue(afwMath.STDEVCLIP))
1494 
1495  self.debugView(ccdExposure, "postISRCCD")
1496 
1497  return pipeBase.Struct(
1498  exposure=ccdExposure,
1499  ossThumb=ossThumb,
1500  flattenedThumb=flattenedThumb,
1501 
1502  preInterpolatedExposure=preInterpExp,
1503  outputExposure=ccdExposure,
1504  outputOssThumbnail=ossThumb,
1505  outputFlattenedThumbnail=flattenedThumb,
1506  )
1507 
1508  @pipeBase.timeMethod
1509  def runDataRef(self, sensorRef):
1510  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1511 
1512  This method contains the `CmdLineTask` interface to the ISR
1513  processing. All IO is handled here, freeing the `run()` method
1514  to manage only pixel-level calculations. The steps performed
1515  are:
1516  - Read in necessary detrending/isr/calibration data.
1517  - Process raw exposure in `run()`.
1518  - Persist the ISR-corrected exposure as "postISRCCD" if
1519  config.doWrite=True.
1520 
1521  Parameters
1522  ----------
1523  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1524  DataRef of the detector data to be processed
1525 
1526  Returns
1527  -------
1528  result : `lsst.pipe.base.Struct`
1529  Result struct with component:
1530  - ``exposure`` : `afw.image.Exposure`
1531  The fully ISR corrected exposure.
1532 
1533  Raises
1534  ------
1535  RuntimeError
1536  Raised if a configuration option is set to True, but the
1537  required calibration data does not exist.
1538 
1539  """
1540  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1541 
1542  ccdExposure = sensorRef.get(self.config.datasetType)
1543 
1544  camera = sensorRef.get("camera")
1545  isrData = self.readIsrData(sensorRef, ccdExposure)
1546 
1547  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1548 
1549  if self.config.doWrite:
1550  sensorRef.put(result.exposure, "postISRCCD")
1551  if result.preInterpolatedExposure is not None:
1552  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1553  if result.ossThumb is not None:
1554  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1555  if result.flattenedThumb is not None:
1556  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1557 
1558  return result
1559 
1560  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1561  """!Retrieve a calibration dataset for removing instrument signature.
1562 
1563  Parameters
1564  ----------
1565 
1566  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1567  DataRef of the detector data to find calibration datasets
1568  for.
1569  datasetType : `str`
1570  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1571  dateObs : `str`, optional
1572  Date of the observation. Used to correct butler failures
1573  when using fallback filters.
1574  immediate : `Bool`
1575  If True, disable butler proxies to enable error handling
1576  within this routine.
1577 
1578  Returns
1579  -------
1580  exposure : `lsst.afw.image.Exposure`
1581  Requested calibration frame.
1582 
1583  Raises
1584  ------
1585  RuntimeError
1586  Raised if no matching calibration frame can be found.
1587  """
1588  try:
1589  exp = dataRef.get(datasetType, immediate=immediate)
1590  except Exception as exc1:
1591  if not self.config.fallbackFilterName:
1592  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1593  try:
1594  if self.config.useFallbackDate and dateObs:
1595  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1596  dateObs=dateObs, immediate=immediate)
1597  else:
1598  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1599  except Exception as exc2:
1600  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1601  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1602  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1603 
1604  if self.config.doAssembleIsrExposures:
1605  exp = self.assembleCcd.assembleCcd(exp)
1606  return exp
1607 
1608  def ensureExposure(self, inputExp, camera, detectorNum):
1609  """Ensure that the data returned by Butler is a fully constructed exposure.
1610 
1611  ISR requires exposure-level image data for historical reasons, so if we did
1612  not recieve that from Butler, construct it from what we have, modifying the
1613  input in place.
1614 
1615  Parameters
1616  ----------
1617  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1618  `lsst.afw.image.ImageF`
1619  The input data structure obtained from Butler.
1620  camera : `lsst.afw.cameraGeom.camera`
1621  The camera associated with the image. Used to find the appropriate
1622  detector.
1623  detectorNum : `int`
1624  The detector this exposure should match.
1625 
1626  Returns
1627  -------
1628  inputExp : `lsst.afw.image.Exposure`
1629  The re-constructed exposure, with appropriate detector parameters.
1630 
1631  Raises
1632  ------
1633  TypeError
1634  Raised if the input data cannot be used to construct an exposure.
1635  """
1636  if isinstance(inputExp, afwImage.DecoratedImageU):
1637  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1638  elif isinstance(inputExp, afwImage.ImageF):
1639  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1640  elif isinstance(inputExp, afwImage.MaskedImageF):
1641  inputExp = afwImage.makeExposure(inputExp)
1642  elif isinstance(inputExp, afwImage.Exposure):
1643  pass
1644  elif inputExp is None:
1645  # Assume this will be caught by the setup if it is a problem.
1646  return inputExp
1647  else:
1648  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1649  (type(inputExp), ))
1650 
1651  if inputExp.getDetector() is None:
1652  inputExp.setDetector(camera[detectorNum])
1653 
1654  return inputExp
1655 
1656  def convertIntToFloat(self, exposure):
1657  """Convert exposure image from uint16 to float.
1658 
1659  If the exposure does not need to be converted, the input is
1660  immediately returned. For exposures that are converted to use
1661  floating point pixels, the variance is set to unity and the
1662  mask to zero.
1663 
1664  Parameters
1665  ----------
1666  exposure : `lsst.afw.image.Exposure`
1667  The raw exposure to be converted.
1668 
1669  Returns
1670  -------
1671  newexposure : `lsst.afw.image.Exposure`
1672  The input ``exposure``, converted to floating point pixels.
1673 
1674  Raises
1675  ------
1676  RuntimeError
1677  Raised if the exposure type cannot be converted to float.
1678 
1679  """
1680  if isinstance(exposure, afwImage.ExposureF):
1681  # Nothing to be done
1682  self.log.debug("Exposure already of type float.")
1683  return exposure
1684  if not hasattr(exposure, "convertF"):
1685  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1686 
1687  newexposure = exposure.convertF()
1688  newexposure.variance[:] = 1
1689  newexposure.mask[:] = 0x0
1690 
1691  return newexposure
1692 
1693  def maskAmplifier(self, ccdExposure, amp, defects):
1694  """Identify bad amplifiers, saturated and suspect pixels.
1695 
1696  Parameters
1697  ----------
1698  ccdExposure : `lsst.afw.image.Exposure`
1699  Input exposure to be masked.
1700  amp : `lsst.afw.table.AmpInfoCatalog`
1701  Catalog of parameters defining the amplifier on this
1702  exposure to mask.
1703  defects : `lsst.meas.algorithms.Defects`
1704  List of defects. Used to determine if the entire
1705  amplifier is bad.
1706 
1707  Returns
1708  -------
1709  badAmp : `Bool`
1710  If this is true, the entire amplifier area is covered by
1711  defects and unusable.
1712 
1713  """
1714  maskedImage = ccdExposure.getMaskedImage()
1715 
1716  badAmp = False
1717 
1718  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1719  # comparison with current defects definition.
1720  if defects is not None:
1721  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1722 
1723  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1724  # association with pixels in current ccdExposure).
1725  if badAmp:
1726  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1727  afwImage.PARENT)
1728  maskView = dataView.getMask()
1729  maskView |= maskView.getPlaneBitMask("BAD")
1730  del maskView
1731  return badAmp
1732 
1733  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1734  # Saturation and suspect pixels can be masked now, though.
1735  limits = dict()
1736  if self.config.doSaturation and not badAmp:
1737  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1738  if self.config.doSuspect and not badAmp:
1739  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1740  if math.isfinite(self.config.saturation):
1741  limits.update({self.config.saturatedMaskName: self.config.saturation})
1742 
1743  for maskName, maskThreshold in limits.items():
1744  if not math.isnan(maskThreshold):
1745  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1746  isrFunctions.makeThresholdMask(
1747  maskedImage=dataView,
1748  threshold=maskThreshold,
1749  growFootprints=0,
1750  maskName=maskName
1751  )
1752 
1753  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1754  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1755  afwImage.PARENT)
1756  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1757  self.config.suspectMaskName])
1758  if numpy.all(maskView.getArray() & maskVal > 0):
1759  badAmp = True
1760  maskView |= maskView.getPlaneBitMask("BAD")
1761 
1762  return badAmp
1763 
1764  def overscanCorrection(self, ccdExposure, amp):
1765  """Apply overscan correction in place.
1766 
1767  This method does initial pixel rejection of the overscan
1768  region. The overscan can also be optionally segmented to
1769  allow for discontinuous overscan responses to be fit
1770  separately. The actual overscan subtraction is performed by
1771  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1772  which is called here after the amplifier is preprocessed.
1773 
1774  Parameters
1775  ----------
1776  ccdExposure : `lsst.afw.image.Exposure`
1777  Exposure to have overscan correction performed.
1778  amp : `lsst.afw.table.AmpInfoCatalog`
1779  The amplifier to consider while correcting the overscan.
1780 
1781  Returns
1782  -------
1783  overscanResults : `lsst.pipe.base.Struct`
1784  Result struct with components:
1785  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1786  Value or fit subtracted from the amplifier image data.
1787  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1788  Value or fit subtracted from the overscan image data.
1789  - ``overscanImage`` : `lsst.afw.image.Image`
1790  Image of the overscan region with the overscan
1791  correction applied. This quantity is used to estimate
1792  the amplifier read noise empirically.
1793 
1794  Raises
1795  ------
1796  RuntimeError
1797  Raised if the ``amp`` does not contain raw pixel information.
1798 
1799  See Also
1800  --------
1801  lsst.ip.isr.isrFunctions.overscanCorrection
1802  """
1803  if not amp.getHasRawInfo():
1804  raise RuntimeError("This method must be executed on an amp with raw information.")
1805 
1806  if amp.getRawHorizontalOverscanBBox().isEmpty():
1807  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1808  return None
1809 
1810  statControl = afwMath.StatisticsControl()
1811  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1812 
1813  # Determine the bounding boxes
1814  dataBBox = amp.getRawDataBBox()
1815  oscanBBox = amp.getRawHorizontalOverscanBBox()
1816  dx0 = 0
1817  dx1 = 0
1818 
1819  prescanBBox = amp.getRawPrescanBBox()
1820  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1821  dx0 += self.config.overscanNumLeadingColumnsToSkip
1822  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1823  else:
1824  dx0 += self.config.overscanNumTrailingColumnsToSkip
1825  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1826 
1827  # Determine if we need to work on subregions of the amplifier and overscan.
1828  imageBBoxes = []
1829  overscanBBoxes = []
1830 
1831  if ((self.config.overscanBiasJump and
1832  self.config.overscanBiasJumpLocation) and
1833  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1834  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1835  self.config.overscanBiasJumpDevices)):
1836  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1837  yLower = self.config.overscanBiasJumpLocation
1838  yUpper = dataBBox.getHeight() - yLower
1839  else:
1840  yUpper = self.config.overscanBiasJumpLocation
1841  yLower = dataBBox.getHeight() - yUpper
1842 
1843  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1844  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1845  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1846  lsst.geom.Extent2I(dx0, 0),
1847  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1848  yLower)))
1849 
1850  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1851  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1852  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1853  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1854  yUpper)))
1855  else:
1856  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1857  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1858  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1859  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1860  oscanBBox.getHeight())))
1861 
1862  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1863  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1864  ampImage = ccdExposure.maskedImage[imageBBox]
1865  overscanImage = ccdExposure.maskedImage[overscanBBox]
1866 
1867  overscanArray = overscanImage.image.array
1868  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1869  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1870  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1871 
1872  statControl = afwMath.StatisticsControl()
1873  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1874 
1875  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1876  overscanImage=overscanImage,
1877  fitType=self.config.overscanFitType,
1878  order=self.config.overscanOrder,
1879  collapseRej=self.config.overscanNumSigmaClip,
1880  statControl=statControl,
1881  overscanIsInt=self.config.overscanIsInt
1882  )
1883 
1884  # Measure average overscan levels and record them in the metadata.
1885  levelStat = afwMath.MEDIAN
1886  sigmaStat = afwMath.STDEVCLIP
1887 
1888  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1889  self.config.qa.flatness.nIter)
1890  metadata = ccdExposure.getMetadata()
1891  ampNum = amp.getName()
1892  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1893  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1894  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1895  else:
1896  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1897  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1898  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1899 
1900  return overscanResults
1901 
1902  def updateVariance(self, ampExposure, amp, overscanImage=None):
1903  """Set the variance plane using the amplifier gain and read noise
1904 
1905  The read noise is calculated from the ``overscanImage`` if the
1906  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1907  the value from the amplifier data is used.
1908 
1909  Parameters
1910  ----------
1911  ampExposure : `lsst.afw.image.Exposure`
1912  Exposure to process.
1913  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1914  Amplifier detector data.
1915  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1916  Image of overscan, required only for empirical read noise.
1917 
1918  See also
1919  --------
1920  lsst.ip.isr.isrFunctions.updateVariance
1921  """
1922  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1923  gain = amp.getGain()
1924 
1925  if math.isnan(gain):
1926  gain = 1.0
1927  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1928  elif gain <= 0:
1929  patchedGain = 1.0
1930  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1931  amp.getName(), gain, patchedGain)
1932  gain = patchedGain
1933 
1934  if self.config.doEmpiricalReadNoise and overscanImage is None:
1935  self.log.info("Overscan is none for EmpiricalReadNoise.")
1936 
1937  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1938  stats = afwMath.StatisticsControl()
1939  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1940  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1941  self.log.info("Calculated empirical read noise for amp %s: %f.",
1942  amp.getName(), readNoise)
1943  else:
1944  readNoise = amp.getReadNoise()
1945 
1946  isrFunctions.updateVariance(
1947  maskedImage=ampExposure.getMaskedImage(),
1948  gain=gain,
1949  readNoise=readNoise,
1950  )
1951 
1952  def darkCorrection(self, exposure, darkExposure, invert=False):
1953  """!Apply dark correction in place.
1954 
1955  Parameters
1956  ----------
1957  exposure : `lsst.afw.image.Exposure`
1958  Exposure to process.
1959  darkExposure : `lsst.afw.image.Exposure`
1960  Dark exposure of the same size as ``exposure``.
1961  invert : `Bool`, optional
1962  If True, re-add the dark to an already corrected image.
1963 
1964  Raises
1965  ------
1966  RuntimeError
1967  Raised if either ``exposure`` or ``darkExposure`` do not
1968  have their dark time defined.
1969 
1970  See Also
1971  --------
1972  lsst.ip.isr.isrFunctions.darkCorrection
1973  """
1974  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1975  if math.isnan(expScale):
1976  raise RuntimeError("Exposure darktime is NAN.")
1977  if darkExposure.getInfo().getVisitInfo() is not None:
1978  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1979  else:
1980  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1981  # so getDarkTime() does not exist.
1982  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1983  darkScale = 1.0
1984 
1985  if math.isnan(darkScale):
1986  raise RuntimeError("Dark calib darktime is NAN.")
1987  isrFunctions.darkCorrection(
1988  maskedImage=exposure.getMaskedImage(),
1989  darkMaskedImage=darkExposure.getMaskedImage(),
1990  expScale=expScale,
1991  darkScale=darkScale,
1992  invert=invert,
1993  trimToFit=self.config.doTrimToMatchCalib
1994  )
1995 
1996  def doLinearize(self, detector):
1997  """!Check if linearization is needed for the detector cameraGeom.
1998 
1999  Checks config.doLinearize and the linearity type of the first
2000  amplifier.
2001 
2002  Parameters
2003  ----------
2004  detector : `lsst.afw.cameraGeom.Detector`
2005  Detector to get linearity type from.
2006 
2007  Returns
2008  -------
2009  doLinearize : `Bool`
2010  If True, linearization should be performed.
2011  """
2012  return self.config.doLinearize and \
2013  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2014 
2015  def flatCorrection(self, exposure, flatExposure, invert=False):
2016  """!Apply flat correction in place.
2017 
2018  Parameters
2019  ----------
2020  exposure : `lsst.afw.image.Exposure`
2021  Exposure to process.
2022  flatExposure : `lsst.afw.image.Exposure`
2023  Flat exposure of the same size as ``exposure``.
2024  invert : `Bool`, optional
2025  If True, unflatten an already flattened image.
2026 
2027  See Also
2028  --------
2029  lsst.ip.isr.isrFunctions.flatCorrection
2030  """
2031  isrFunctions.flatCorrection(
2032  maskedImage=exposure.getMaskedImage(),
2033  flatMaskedImage=flatExposure.getMaskedImage(),
2034  scalingType=self.config.flatScalingType,
2035  userScale=self.config.flatUserScale,
2036  invert=invert,
2037  trimToFit=self.config.doTrimToMatchCalib
2038  )
2039 
2040  def saturationDetection(self, exposure, amp):
2041  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2042 
2043  Parameters
2044  ----------
2045  exposure : `lsst.afw.image.Exposure`
2046  Exposure to process. Only the amplifier DataSec is processed.
2047  amp : `lsst.afw.table.AmpInfoCatalog`
2048  Amplifier detector data.
2049 
2050  See Also
2051  --------
2052  lsst.ip.isr.isrFunctions.makeThresholdMask
2053  """
2054  if not math.isnan(amp.getSaturation()):
2055  maskedImage = exposure.getMaskedImage()
2056  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2057  isrFunctions.makeThresholdMask(
2058  maskedImage=dataView,
2059  threshold=amp.getSaturation(),
2060  growFootprints=0,
2061  maskName=self.config.saturatedMaskName,
2062  )
2063 
2064  def saturationInterpolation(self, exposure):
2065  """!Interpolate over saturated pixels, in place.
2066 
2067  This method should be called after `saturationDetection`, to
2068  ensure that the saturated pixels have been identified in the
2069  SAT mask. It should also be called after `assembleCcd`, since
2070  saturated regions may cross amplifier boundaries.
2071 
2072  Parameters
2073  ----------
2074  exposure : `lsst.afw.image.Exposure`
2075  Exposure to process.
2076 
2077  See Also
2078  --------
2079  lsst.ip.isr.isrTask.saturationDetection
2080  lsst.ip.isr.isrFunctions.interpolateFromMask
2081  """
2082  isrFunctions.interpolateFromMask(
2083  maskedImage=exposure.getMaskedImage(),
2084  fwhm=self.config.fwhm,
2085  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2086  maskNameList=list(self.config.saturatedMaskName),
2087  )
2088 
2089  def suspectDetection(self, exposure, amp):
2090  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2091 
2092  Parameters
2093  ----------
2094  exposure : `lsst.afw.image.Exposure`
2095  Exposure to process. Only the amplifier DataSec is processed.
2096  amp : `lsst.afw.table.AmpInfoCatalog`
2097  Amplifier detector data.
2098 
2099  See Also
2100  --------
2101  lsst.ip.isr.isrFunctions.makeThresholdMask
2102 
2103  Notes
2104  -----
2105  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2106  This is intended to indicate pixels that may be affected by unknown systematics;
2107  for example if non-linearity corrections above a certain level are unstable
2108  then that would be a useful value for suspectLevel. A value of `nan` indicates
2109  that no such level exists and no pixels are to be masked as suspicious.
2110  """
2111  suspectLevel = amp.getSuspectLevel()
2112  if math.isnan(suspectLevel):
2113  return
2114 
2115  maskedImage = exposure.getMaskedImage()
2116  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2117  isrFunctions.makeThresholdMask(
2118  maskedImage=dataView,
2119  threshold=suspectLevel,
2120  growFootprints=0,
2121  maskName=self.config.suspectMaskName,
2122  )
2123 
2124  def maskDefect(self, exposure, defectBaseList):
2125  """!Mask defects using mask plane "BAD", in place.
2126 
2127  Parameters
2128  ----------
2129  exposure : `lsst.afw.image.Exposure`
2130  Exposure to process.
2131  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2132  `lsst.afw.image.DefectBase`.
2133  List of defects to mask.
2134 
2135  Notes
2136  -----
2137  Call this after CCD assembly, since defects may cross amplifier boundaries.
2138  """
2139  maskedImage = exposure.getMaskedImage()
2140  if not isinstance(defectBaseList, Defects):
2141  # Promotes DefectBase to Defect
2142  defectList = Defects(defectBaseList)
2143  else:
2144  defectList = defectBaseList
2145  defectList.maskPixels(maskedImage, maskName="BAD")
2146 
2147  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2148  """!Mask edge pixels with applicable mask plane.
2149 
2150  Parameters
2151  ----------
2152  exposure : `lsst.afw.image.Exposure`
2153  Exposure to process.
2154  numEdgePixels : `int`, optional
2155  Number of edge pixels to mask.
2156  maskPlane : `str`, optional
2157  Mask plane name to use.
2158  """
2159  maskedImage = exposure.getMaskedImage()
2160  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2161 
2162  if numEdgePixels > 0:
2163  goodBBox = maskedImage.getBBox()
2164  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2165  goodBBox.grow(-numEdgePixels)
2166  # Mask pixels outside goodBBox
2167  SourceDetectionTask.setEdgeBits(
2168  maskedImage,
2169  goodBBox,
2170  maskBitMask
2171  )
2172 
2173  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2174  """Mask and interpolate defects using mask plane "BAD", in place.
2175 
2176  Parameters
2177  ----------
2178  exposure : `lsst.afw.image.Exposure`
2179  Exposure to process.
2180  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2181  `lsst.afw.image.DefectBase`.
2182  List of defects to mask and interpolate.
2183 
2184  See Also
2185  --------
2186  lsst.ip.isr.isrTask.maskDefect()
2187  """
2188  self.maskDefect(exposure, defectBaseList)
2189  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2190  maskPlane="SUSPECT")
2191  isrFunctions.interpolateFromMask(
2192  maskedImage=exposure.getMaskedImage(),
2193  fwhm=self.config.fwhm,
2194  growSaturatedFootprints=0,
2195  maskNameList=["BAD"],
2196  )
2197 
2198  def maskNan(self, exposure):
2199  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2200 
2201  Parameters
2202  ----------
2203  exposure : `lsst.afw.image.Exposure`
2204  Exposure to process.
2205 
2206  Notes
2207  -----
2208  We mask over all NaNs, including those that are masked with
2209  other bits (because those may or may not be interpolated over
2210  later, and we want to remove all NaNs). Despite this
2211  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2212  the historical name.
2213  """
2214  maskedImage = exposure.getMaskedImage()
2215 
2216  # Find and mask NaNs
2217  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2218  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2219  numNans = maskNans(maskedImage, maskVal)
2220  self.metadata.set("NUMNANS", numNans)
2221  if numNans > 0:
2222  self.log.warn("There were %d unmasked NaNs.", numNans)
2223 
2224  def maskAndInterpolateNan(self, exposure):
2225  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2226 
2227  Parameters
2228  ----------
2229  exposure : `lsst.afw.image.Exposure`
2230  Exposure to process.
2231 
2232  See Also
2233  --------
2234  lsst.ip.isr.isrTask.maskNan()
2235  """
2236  self.maskNan(exposure)
2237  isrFunctions.interpolateFromMask(
2238  maskedImage=exposure.getMaskedImage(),
2239  fwhm=self.config.fwhm,
2240  growSaturatedFootprints=0,
2241  maskNameList=["UNMASKEDNAN"],
2242  )
2243 
2244  def measureBackground(self, exposure, IsrQaConfig=None):
2245  """Measure the image background in subgrids, for quality control purposes.
2246 
2247  Parameters
2248  ----------
2249  exposure : `lsst.afw.image.Exposure`
2250  Exposure to process.
2251  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2252  Configuration object containing parameters on which background
2253  statistics and subgrids to use.
2254  """
2255  if IsrQaConfig is not None:
2256  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2257  IsrQaConfig.flatness.nIter)
2258  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2259  statsControl.setAndMask(maskVal)
2260  maskedImage = exposure.getMaskedImage()
2261  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2262  skyLevel = stats.getValue(afwMath.MEDIAN)
2263  skySigma = stats.getValue(afwMath.STDEVCLIP)
2264  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2265  metadata = exposure.getMetadata()
2266  metadata.set('SKYLEVEL', skyLevel)
2267  metadata.set('SKYSIGMA', skySigma)
2268 
2269  # calcluating flatlevel over the subgrids
2270  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2271  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2272  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2273  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2274  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2275  skyLevels = numpy.zeros((nX, nY))
2276 
2277  for j in range(nY):
2278  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2279  for i in range(nX):
2280  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2281 
2282  xLLC = xc - meshXHalf
2283  yLLC = yc - meshYHalf
2284  xURC = xc + meshXHalf - 1
2285  yURC = yc + meshYHalf - 1
2286 
2287  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2288  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2289 
2290  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2291 
2292  good = numpy.where(numpy.isfinite(skyLevels))
2293  skyMedian = numpy.median(skyLevels[good])
2294  flatness = (skyLevels[good] - skyMedian) / skyMedian
2295  flatness_rms = numpy.std(flatness)
2296  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2297 
2298  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2299  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2300  nX, nY, flatness_pp, flatness_rms)
2301 
2302  metadata.set('FLATNESS_PP', float(flatness_pp))
2303  metadata.set('FLATNESS_RMS', float(flatness_rms))
2304  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2305  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2306  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2307 
2308  def roughZeroPoint(self, exposure):
2309  """Set an approximate magnitude zero point for the exposure.
2310 
2311  Parameters
2312  ----------
2313  exposure : `lsst.afw.image.Exposure`
2314  Exposure to process.
2315  """
2316  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2317  if filterName in self.config.fluxMag0T1:
2318  fluxMag0 = self.config.fluxMag0T1[filterName]
2319  else:
2320  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2321  fluxMag0 = self.config.defaultFluxMag0T1
2322 
2323  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2324  if not expTime > 0: # handle NaN as well as <= 0
2325  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2326  return
2327 
2328  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2329  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2330 
2331  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2332  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2333 
2334  Parameters
2335  ----------
2336  ccdExposure : `lsst.afw.image.Exposure`
2337  Exposure to process.
2338  fpPolygon : `lsst.afw.geom.Polygon`
2339  Polygon in focal plane coordinates.
2340  """
2341  # Get ccd corners in focal plane coordinates
2342  ccd = ccdExposure.getDetector()
2343  fpCorners = ccd.getCorners(FOCAL_PLANE)
2344  ccdPolygon = Polygon(fpCorners)
2345 
2346  # Get intersection of ccd corners with fpPolygon
2347  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2348 
2349  # Transform back to pixel positions and build new polygon
2350  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2351  validPolygon = Polygon(ccdPoints)
2352  ccdExposure.getInfo().setValidPolygon(validPolygon)
2353 
2354  @contextmanager
2355  def flatContext(self, exp, flat, dark=None):
2356  """Context manager that applies and removes flats and darks,
2357  if the task is configured to apply them.
2358 
2359  Parameters
2360  ----------
2361  exp : `lsst.afw.image.Exposure`
2362  Exposure to process.
2363  flat : `lsst.afw.image.Exposure`
2364  Flat exposure the same size as ``exp``.
2365  dark : `lsst.afw.image.Exposure`, optional
2366  Dark exposure the same size as ``exp``.
2367 
2368  Yields
2369  ------
2370  exp : `lsst.afw.image.Exposure`
2371  The flat and dark corrected exposure.
2372  """
2373  if self.config.doDark and dark is not None:
2374  self.darkCorrection(exp, dark)
2375  if self.config.doFlat:
2376  self.flatCorrection(exp, flat)
2377  try:
2378  yield exp
2379  finally:
2380  if self.config.doFlat:
2381  self.flatCorrection(exp, flat, invert=True)
2382  if self.config.doDark and dark is not None:
2383  self.darkCorrection(exp, dark, invert=True)
2384 
2385  def debugView(self, exposure, stepname):
2386  """Utility function to examine ISR exposure at different stages.
2387 
2388  Parameters
2389  ----------
2390  exposure : `lsst.afw.image.Exposure`
2391  Exposure to view.
2392  stepname : `str`
2393  State of processing to view.
2394  """
2395  frame = getDebugFrame(self._display, stepname)
2396  if frame:
2397  display = getDisplay(frame)
2398  display.scale('asinh', 'zscale')
2399  display.mtv(exposure)
2400  prompt = "Press Enter to continue [c]... "
2401  while True:
2402  ans = input(prompt).lower()
2403  if ans in ("", "c",):
2404  break
2405 
2406 
2407 class FakeAmp(object):
2408  """A Detector-like object that supports returning gain and saturation level
2409 
2410  This is used when the input exposure does not have a detector.
2411 
2412  Parameters
2413  ----------
2414  exposure : `lsst.afw.image.Exposure`
2415  Exposure to generate a fake amplifier for.
2416  config : `lsst.ip.isr.isrTaskConfig`
2417  Configuration to apply to the fake amplifier.
2418  """
2419 
2420  def __init__(self, exposure, config):
2421  self._bbox = exposure.getBBox(afwImage.LOCAL)
2423  self._gain = config.gain
2424  self._readNoise = config.readNoise
2425  self._saturation = config.saturation
2426 
2427  def getBBox(self):
2428  return self._bbox
2429 
2430  def getRawBBox(self):
2431  return self._bbox
2432 
2433  def getHasRawInfo(self):
2434  return True # but see getRawHorizontalOverscanBBox()
2435 
2437  return self._RawHorizontalOverscanBBox
2438 
2439  def getGain(self):
2440  return self._gain
2441 
2442  def getReadNoise(self):
2443  return self._readNoise
2444 
2445  def getSaturation(self):
2446  return self._saturation
2447 
2448  def getSuspectLevel(self):
2449  return float("NaN")
2450 
2451 
2452 class RunIsrConfig(pexConfig.Config):
2453  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2454 
2455 
2456 class RunIsrTask(pipeBase.CmdLineTask):
2457  """Task to wrap the default IsrTask to allow it to be retargeted.
2458 
2459  The standard IsrTask can be called directly from a command line
2460  program, but doing so removes the ability of the task to be
2461  retargeted. As most cameras override some set of the IsrTask
2462  methods, this would remove those data-specific methods in the
2463  output post-ISR images. This wrapping class fixes the issue,
2464  allowing identical post-ISR images to be generated by both the
2465  processCcd and isrTask code.
2466  """
2467  ConfigClass = RunIsrConfig
2468  _DefaultName = "runIsr"
2469 
2470  def __init__(self, *args, **kwargs):
2471  super().__init__(*args, **kwargs)
2472  self.makeSubtask("isr")
2473 
2474  def runDataRef(self, dataRef):
2475  """
2476  Parameters
2477  ----------
2478  dataRef : `lsst.daf.persistence.ButlerDataRef`
2479  data reference of the detector data to be processed
2480 
2481  Returns
2482  -------
2483  result : `pipeBase.Struct`
2484  Result struct with component:
2485 
2486  - exposure : `lsst.afw.image.Exposure`
2487  Post-ISR processed exposure.
2488  """
2489  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1509
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2244
def debugView(self, exposure, stepname)
Definition: isrTask.py:2385
def __init__(self, kwargs)
Definition: isrTask.py:810
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1608
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:888
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2147
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:819
def runDataRef(self, dataRef)
Definition: isrTask.py:2474
def __init__(self, args, kwargs)
Definition: isrTask.py:2470
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2308
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2173
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2436
def maskNan(self, exposure)
Definition: isrTask.py:2198
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2124
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1764
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1656
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:2015
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1054
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1952
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1996
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2331
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1693
def __init__(self, config=None)
Definition: isrTask.py:179
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2355
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1560
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1902
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2224
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2089
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2064
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:2040
def __init__(self, exposure, config)
Definition: isrTask.py:2420