lsst.ip.isr  19.0.0-8-g2034a75+3
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.PrerequisiteInput(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ImageF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ImageF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="MaskedImageF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  defects = cT.PrerequisiteInput(
112  name='defects',
113  doc="Input defect tables.",
114  storageClass="DefectsList",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  opticsTransmission = cT.PrerequisiteInput(
118  name="transmission_optics",
119  storageClass="TransmissionCurve",
120  doc="Transmission curve due to the optics.",
121  dimensions=["instrument", "calibration_label"],
122  )
123  filterTransmission = cT.PrerequisiteInput(
124  name="transmission_filter",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the filter.",
127  dimensions=["instrument", "physical_filter", "calibration_label"],
128  )
129  sensorTransmission = cT.PrerequisiteInput(
130  name="transmission_sensor",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the sensor.",
133  dimensions=["instrument", "calibration_label", "detector"],
134  )
135  atmosphereTransmission = cT.PrerequisiteInput(
136  name="transmission_atmosphere",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the atmosphere.",
139  dimensions=["instrument"],
140  )
141  illumMaskedImage = cT.PrerequisiteInput(
142  name="illum",
143  doc="Input illumination correction.",
144  storageClass="MaskedImageF",
145  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
146  )
147 
148  outputExposure = cT.Output(
149  name='postISRCCD',
150  doc="Output ISR processed exposure.",
151  storageClass="ExposureF",
152  dimensions=["instrument", "visit", "detector"],
153  )
154  preInterpExposure = cT.Output(
155  name='preInterpISRCCD',
156  doc="Output ISR processed exposure, with pixels left uninterpolated.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "visit", "detector"],
159  )
160  outputOssThumbnail = cT.Output(
161  name="OssThumb",
162  doc="Output Overscan-subtracted thumbnail image.",
163  storageClass="Thumbnail",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputFlattenedThumbnail = cT.Output(
167  name="FlattenedThumb",
168  doc="Output flat-corrected thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "visit", "detector"],
171  )
172 
173  def __init__(self, *, config=None):
174  super().__init__(config=config)
175 
176  if config.doBias is not True:
177  self.prerequisiteInputs.discard("bias")
178  if config.doLinearize is not True:
179  self.prerequisiteInputs.discard("linearizer")
180  if config.doCrosstalk is not True:
181  self.prerequisiteInputs.discard("crosstalkSources")
182  if config.doBrighterFatter is not True:
183  self.prerequisiteInputs.discard("bfKernel")
184  if config.doDefect is not True:
185  self.prerequisiteInputs.discard("defects")
186  if config.doDark is not True:
187  self.prerequisiteInputs.discard("dark")
188  if config.doFlat is not True:
189  self.prerequisiteInputs.discard("flat")
190  if config.doAttachTransmissionCurve is not True:
191  self.prerequisiteInputs.discard("opticsTransmission")
192  self.prerequisiteInputs.discard("filterTransmission")
193  self.prerequisiteInputs.discard("sensorTransmission")
194  self.prerequisiteInputs.discard("atmosphereTransmission")
195  if config.doUseOpticsTransmission is not True:
196  self.prerequisiteInputs.discard("opticsTransmission")
197  if config.doUseFilterTransmission is not True:
198  self.prerequisiteInputs.discard("filterTransmission")
199  if config.doUseSensorTransmission is not True:
200  self.prerequisiteInputs.discard("sensorTransmission")
201  if config.doUseAtmosphereTransmission is not True:
202  self.prerequisiteInputs.discard("atmosphereTransmission")
203  if config.doIlluminationCorrection is not True:
204  self.prerequisiteInputs.discard("illumMaskedImage")
205 
206  if config.doWrite is not True:
207  self.outputs.discard("outputExposure")
208  self.outputs.discard("preInterpExposure")
209  self.outputs.discard("outputFlattenedThumbnail")
210  self.outputs.discard("outputOssThumbnail")
211  if config.doSaveInterpPixels is not True:
212  self.outputs.discard("preInterpExposure")
213  if config.qa.doThumbnailOss is not True:
214  self.outputs.discard("outputOssThumbnail")
215  if config.qa.doThumbnailFlattened is not True:
216  self.outputs.discard("outputFlattenedThumbnail")
217 
218 
219 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
220  pipelineConnections=IsrTaskConnections):
221  """Configuration parameters for IsrTask.
222 
223  Items are grouped in the order in which they are executed by the task.
224  """
225  datasetType = pexConfig.Field(
226  dtype=str,
227  doc="Dataset type for input data; users will typically leave this alone, "
228  "but camera-specific ISR tasks will override it",
229  default="raw",
230  )
231 
232  fallbackFilterName = pexConfig.Field(
233  dtype=str,
234  doc="Fallback default filter name for calibrations.",
235  optional=True
236  )
237  useFallbackDate = pexConfig.Field(
238  dtype=bool,
239  doc="Pass observation date when using fallback filter.",
240  default=False,
241  )
242  expectWcs = pexConfig.Field(
243  dtype=bool,
244  default=True,
245  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
246  )
247  fwhm = pexConfig.Field(
248  dtype=float,
249  doc="FWHM of PSF in arcseconds.",
250  default=1.0,
251  )
252  qa = pexConfig.ConfigField(
253  dtype=isrQa.IsrQaConfig,
254  doc="QA related configuration options.",
255  )
256 
257  # Image conversion configuration
258  doConvertIntToFloat = pexConfig.Field(
259  dtype=bool,
260  doc="Convert integer raw images to floating point values?",
261  default=True,
262  )
263 
264  # Saturated pixel handling.
265  doSaturation = pexConfig.Field(
266  dtype=bool,
267  doc="Mask saturated pixels? NB: this is totally independent of the"
268  " interpolation option - this is ONLY setting the bits in the mask."
269  " To have them interpolated make sure doSaturationInterpolation=True",
270  default=True,
271  )
272  saturatedMaskName = pexConfig.Field(
273  dtype=str,
274  doc="Name of mask plane to use in saturation detection and interpolation",
275  default="SAT",
276  )
277  saturation = pexConfig.Field(
278  dtype=float,
279  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
280  default=float("NaN"),
281  )
282  growSaturationFootprintSize = pexConfig.Field(
283  dtype=int,
284  doc="Number of pixels by which to grow the saturation footprints",
285  default=1,
286  )
287 
288  # Suspect pixel handling.
289  doSuspect = pexConfig.Field(
290  dtype=bool,
291  doc="Mask suspect pixels?",
292  default=False,
293  )
294  suspectMaskName = pexConfig.Field(
295  dtype=str,
296  doc="Name of mask plane to use for suspect pixels",
297  default="SUSPECT",
298  )
299  numEdgeSuspect = pexConfig.Field(
300  dtype=int,
301  doc="Number of edge pixels to be flagged as untrustworthy.",
302  default=0,
303  )
304 
305  # Initial masking options.
306  doSetBadRegions = pexConfig.Field(
307  dtype=bool,
308  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
309  default=True,
310  )
311  badStatistic = pexConfig.ChoiceField(
312  dtype=str,
313  doc="How to estimate the average value for BAD regions.",
314  default='MEANCLIP',
315  allowed={
316  "MEANCLIP": "Correct using the (clipped) mean of good data",
317  "MEDIAN": "Correct using the median of the good data",
318  },
319  )
320 
321  # Overscan subtraction configuration.
322  doOverscan = pexConfig.Field(
323  dtype=bool,
324  doc="Do overscan subtraction?",
325  default=True,
326  )
327  overscanFitType = pexConfig.ChoiceField(
328  dtype=str,
329  doc="The method for fitting the overscan bias level.",
330  default='MEDIAN',
331  allowed={
332  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
333  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
334  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
335  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
336  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
337  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
338  "MEAN": "Correct using the mean of the overscan region",
339  "MEANCLIP": "Correct using a clipped mean of the overscan region",
340  "MEDIAN": "Correct using the median of the overscan region",
341  },
342  )
343  overscanOrder = pexConfig.Field(
344  dtype=int,
345  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
346  "or number of spline knots if overscan fit type is a spline."),
347  default=1,
348  )
349  overscanNumSigmaClip = pexConfig.Field(
350  dtype=float,
351  doc="Rejection threshold (sigma) for collapsing overscan before fit",
352  default=3.0,
353  )
354  overscanIsInt = pexConfig.Field(
355  dtype=bool,
356  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
357  default=True,
358  )
359  overscanNumLeadingColumnsToSkip = pexConfig.Field(
360  dtype=int,
361  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
362  default=0,
363  )
364  overscanNumTrailingColumnsToSkip = pexConfig.Field(
365  dtype=int,
366  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
367  default=0,
368  )
369  overscanMaxDev = pexConfig.Field(
370  dtype=float,
371  doc="Maximum deviation from the median for overscan",
372  default=1000.0, check=lambda x: x > 0
373  )
374  overscanBiasJump = pexConfig.Field(
375  dtype=bool,
376  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
377  default=False,
378  )
379  overscanBiasJumpKeyword = pexConfig.Field(
380  dtype=str,
381  doc="Header keyword containing information about devices.",
382  default="NO_SUCH_KEY",
383  )
384  overscanBiasJumpDevices = pexConfig.ListField(
385  dtype=str,
386  doc="List of devices that need piecewise overscan correction.",
387  default=(),
388  )
389  overscanBiasJumpLocation = pexConfig.Field(
390  dtype=int,
391  doc="Location of bias jump along y-axis.",
392  default=0,
393  )
394 
395  # Amplifier to CCD assembly configuration
396  doAssembleCcd = pexConfig.Field(
397  dtype=bool,
398  default=True,
399  doc="Assemble amp-level exposures into a ccd-level exposure?"
400  )
401  assembleCcd = pexConfig.ConfigurableField(
402  target=AssembleCcdTask,
403  doc="CCD assembly task",
404  )
405 
406  # General calibration configuration.
407  doAssembleIsrExposures = pexConfig.Field(
408  dtype=bool,
409  default=False,
410  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
411  )
412  doTrimToMatchCalib = pexConfig.Field(
413  dtype=bool,
414  default=False,
415  doc="Trim raw data to match calibration bounding boxes?"
416  )
417 
418  # Bias subtraction.
419  doBias = pexConfig.Field(
420  dtype=bool,
421  doc="Apply bias frame correction?",
422  default=True,
423  )
424  biasDataProductName = pexConfig.Field(
425  dtype=str,
426  doc="Name of the bias data product",
427  default="bias",
428  )
429 
430  # Variance construction
431  doVariance = pexConfig.Field(
432  dtype=bool,
433  doc="Calculate variance?",
434  default=True
435  )
436  gain = pexConfig.Field(
437  dtype=float,
438  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
439  default=float("NaN"),
440  )
441  readNoise = pexConfig.Field(
442  dtype=float,
443  doc="The read noise to use if no Detector is present in the Exposure",
444  default=0.0,
445  )
446  doEmpiricalReadNoise = pexConfig.Field(
447  dtype=bool,
448  default=False,
449  doc="Calculate empirical read noise instead of value from AmpInfo data?"
450  )
451 
452  # Linearization.
453  doLinearize = pexConfig.Field(
454  dtype=bool,
455  doc="Correct for nonlinearity of the detector's response?",
456  default=True,
457  )
458 
459  # Crosstalk.
460  doCrosstalk = pexConfig.Field(
461  dtype=bool,
462  doc="Apply intra-CCD crosstalk correction?",
463  default=False,
464  )
465  doCrosstalkBeforeAssemble = pexConfig.Field(
466  dtype=bool,
467  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
468  default=False,
469  )
470  crosstalk = pexConfig.ConfigurableField(
471  target=CrosstalkTask,
472  doc="Intra-CCD crosstalk correction",
473  )
474 
475  # Masking options.
476  doDefect = pexConfig.Field(
477  dtype=bool,
478  doc="Apply correction for CCD defects, e.g. hot pixels?",
479  default=True,
480  )
481  doNanMasking = pexConfig.Field(
482  dtype=bool,
483  doc="Mask NAN pixels?",
484  default=True,
485  )
486  doWidenSaturationTrails = pexConfig.Field(
487  dtype=bool,
488  doc="Widen bleed trails based on their width?",
489  default=True
490  )
491 
492  # Brighter-Fatter correction.
493  doBrighterFatter = pexConfig.Field(
494  dtype=bool,
495  default=False,
496  doc="Apply the brighter fatter correction"
497  )
498  brighterFatterLevel = pexConfig.ChoiceField(
499  dtype=str,
500  default="DETECTOR",
501  doc="The level at which to correct for brighter-fatter.",
502  allowed={
503  "AMP": "Every amplifier treated separately.",
504  "DETECTOR": "One kernel per detector",
505  }
506  )
507  brighterFatterMaxIter = pexConfig.Field(
508  dtype=int,
509  default=10,
510  doc="Maximum number of iterations for the brighter fatter correction"
511  )
512  brighterFatterThreshold = pexConfig.Field(
513  dtype=float,
514  default=1000,
515  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
516  " absolute value of the difference between the current corrected image and the one"
517  " from the previous iteration summed over all the pixels."
518  )
519  brighterFatterApplyGain = pexConfig.Field(
520  dtype=bool,
521  default=True,
522  doc="Should the gain be applied when applying the brighter fatter correction?"
523  )
524  brighterFatterMaskGrowSize = pexConfig.Field(
525  dtype=int,
526  default=0,
527  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
528  " when brighter-fatter correction is applied."
529  )
530 
531  # Dark subtraction.
532  doDark = pexConfig.Field(
533  dtype=bool,
534  doc="Apply dark frame correction?",
535  default=True,
536  )
537  darkDataProductName = pexConfig.Field(
538  dtype=str,
539  doc="Name of the dark data product",
540  default="dark",
541  )
542 
543  # Camera-specific stray light removal.
544  doStrayLight = pexConfig.Field(
545  dtype=bool,
546  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
547  default=False,
548  )
549  strayLight = pexConfig.ConfigurableField(
550  target=StrayLightTask,
551  doc="y-band stray light correction"
552  )
553 
554  # Flat correction.
555  doFlat = pexConfig.Field(
556  dtype=bool,
557  doc="Apply flat field correction?",
558  default=True,
559  )
560  flatDataProductName = pexConfig.Field(
561  dtype=str,
562  doc="Name of the flat data product",
563  default="flat",
564  )
565  flatScalingType = pexConfig.ChoiceField(
566  dtype=str,
567  doc="The method for scaling the flat on the fly.",
568  default='USER',
569  allowed={
570  "USER": "Scale by flatUserScale",
571  "MEAN": "Scale by the inverse of the mean",
572  "MEDIAN": "Scale by the inverse of the median",
573  },
574  )
575  flatUserScale = pexConfig.Field(
576  dtype=float,
577  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
578  default=1.0,
579  )
580  doTweakFlat = pexConfig.Field(
581  dtype=bool,
582  doc="Tweak flats to match observed amplifier ratios?",
583  default=False
584  )
585 
586  # Amplifier normalization based on gains instead of using flats configuration.
587  doApplyGains = pexConfig.Field(
588  dtype=bool,
589  doc="Correct the amplifiers for their gains instead of applying flat correction",
590  default=False,
591  )
592  normalizeGains = pexConfig.Field(
593  dtype=bool,
594  doc="Normalize all the amplifiers in each CCD to have the same median value.",
595  default=False,
596  )
597 
598  # Fringe correction.
599  doFringe = pexConfig.Field(
600  dtype=bool,
601  doc="Apply fringe correction?",
602  default=True,
603  )
604  fringe = pexConfig.ConfigurableField(
605  target=FringeTask,
606  doc="Fringe subtraction task",
607  )
608  fringeAfterFlat = pexConfig.Field(
609  dtype=bool,
610  doc="Do fringe subtraction after flat-fielding?",
611  default=True,
612  )
613 
614  # Distortion model application.
615  doAddDistortionModel = pexConfig.Field(
616  dtype=bool,
617  doc="Apply a distortion model based on camera geometry to the WCS?",
618  default=True,
619  deprecated=("Camera geometry is incorporated when reading the raw files."
620  " This option no longer is used, and will be removed after v19.")
621  )
622 
623  # Initial CCD-level background statistics options.
624  doMeasureBackground = pexConfig.Field(
625  dtype=bool,
626  doc="Measure the background level on the reduced image?",
627  default=False,
628  )
629 
630  # Camera-specific masking configuration.
631  doCameraSpecificMasking = pexConfig.Field(
632  dtype=bool,
633  doc="Mask camera-specific bad regions?",
634  default=False,
635  )
636  masking = pexConfig.ConfigurableField(
637  target=MaskingTask,
638  doc="Masking task."
639  )
640 
641  # Interpolation options.
642 
643  doInterpolate = pexConfig.Field(
644  dtype=bool,
645  doc="Interpolate masked pixels?",
646  default=True,
647  )
648  doSaturationInterpolation = pexConfig.Field(
649  dtype=bool,
650  doc="Perform interpolation over pixels masked as saturated?"
651  " NB: This is independent of doSaturation; if that is False this plane"
652  " will likely be blank, resulting in a no-op here.",
653  default=True,
654  )
655  doNanInterpolation = pexConfig.Field(
656  dtype=bool,
657  doc="Perform interpolation over pixels masked as NaN?"
658  " NB: This is independent of doNanMasking; if that is False this plane"
659  " will likely be blank, resulting in a no-op here.",
660  default=True,
661  )
662  doNanInterpAfterFlat = pexConfig.Field(
663  dtype=bool,
664  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
665  "also have to interpolate them before flat-fielding."),
666  default=False,
667  )
668  maskListToInterpolate = pexConfig.ListField(
669  dtype=str,
670  doc="List of mask planes that should be interpolated.",
671  default=['SAT', 'BAD', 'UNMASKEDNAN'],
672  )
673  doSaveInterpPixels = pexConfig.Field(
674  dtype=bool,
675  doc="Save a copy of the pre-interpolated pixel values?",
676  default=False,
677  )
678 
679  # Default photometric calibration options.
680  fluxMag0T1 = pexConfig.DictField(
681  keytype=str,
682  itemtype=float,
683  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
684  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
685  ))
686  )
687  defaultFluxMag0T1 = pexConfig.Field(
688  dtype=float,
689  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
690  default=pow(10.0, 0.4*28.0)
691  )
692 
693  # Vignette correction configuration.
694  doVignette = pexConfig.Field(
695  dtype=bool,
696  doc="Apply vignetting parameters?",
697  default=False,
698  )
699  vignette = pexConfig.ConfigurableField(
700  target=VignetteTask,
701  doc="Vignetting task.",
702  )
703 
704  # Transmission curve configuration.
705  doAttachTransmissionCurve = pexConfig.Field(
706  dtype=bool,
707  default=False,
708  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
709  )
710  doUseOpticsTransmission = pexConfig.Field(
711  dtype=bool,
712  default=True,
713  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
714  )
715  doUseFilterTransmission = pexConfig.Field(
716  dtype=bool,
717  default=True,
718  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
719  )
720  doUseSensorTransmission = pexConfig.Field(
721  dtype=bool,
722  default=True,
723  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
724  )
725  doUseAtmosphereTransmission = pexConfig.Field(
726  dtype=bool,
727  default=True,
728  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
729  )
730 
731  # Illumination correction.
732  doIlluminationCorrection = pexConfig.Field(
733  dtype=bool,
734  default=False,
735  doc="Perform illumination correction?"
736  )
737  illuminationCorrectionDataProductName = pexConfig.Field(
738  dtype=str,
739  doc="Name of the illumination correction data product.",
740  default="illumcor",
741  )
742  illumScale = pexConfig.Field(
743  dtype=float,
744  doc="Scale factor for the illumination correction.",
745  default=1.0,
746  )
747  illumFilters = pexConfig.ListField(
748  dtype=str,
749  default=[],
750  doc="Only perform illumination correction for these filters."
751  )
752 
753  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
754  doWrite = pexConfig.Field(
755  dtype=bool,
756  doc="Persist postISRCCD?",
757  default=True,
758  )
759 
760  def validate(self):
761  super().validate()
762  if self.doFlat and self.doApplyGains:
763  raise ValueError("You may not specify both doFlat and doApplyGains")
764  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
765  self.config.maskListToInterpolate.append("SAT")
766  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
767  self.config.maskListToInterpolate.append("UNMASKEDNAN")
768 
769 
770 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
771  """Apply common instrument signature correction algorithms to a raw frame.
772 
773  The process for correcting imaging data is very similar from
774  camera to camera. This task provides a vanilla implementation of
775  doing these corrections, including the ability to turn certain
776  corrections off if they are not needed. The inputs to the primary
777  method, `run()`, are a raw exposure to be corrected and the
778  calibration data products. The raw input is a single chip sized
779  mosaic of all amps including overscans and other non-science
780  pixels. The method `runDataRef()` identifies and defines the
781  calibration data products, and is intended for use by a
782  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
783  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
784  subclassed for different camera, although the most camera specific
785  methods have been split into subtasks that can be redirected
786  appropriately.
787 
788  The __init__ method sets up the subtasks for ISR processing, using
789  the defaults from `lsst.ip.isr`.
790 
791  Parameters
792  ----------
793  args : `list`
794  Positional arguments passed to the Task constructor. None used at this time.
795  kwargs : `dict`, optional
796  Keyword arguments passed on to the Task constructor. None used at this time.
797  """
798  ConfigClass = IsrTaskConfig
799  _DefaultName = "isr"
800 
801  def __init__(self, **kwargs):
802  super().__init__(**kwargs)
803  self.makeSubtask("assembleCcd")
804  self.makeSubtask("crosstalk")
805  self.makeSubtask("strayLight")
806  self.makeSubtask("fringe")
807  self.makeSubtask("masking")
808  self.makeSubtask("vignette")
809 
810  def runQuantum(self, butlerQC, inputRefs, outputRefs):
811  inputs = butlerQC.get(inputRefs)
812 
813  try:
814  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
815  except Exception as e:
816  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
817  (inputRefs, e))
818 
819  inputs['isGen3'] = True
820 
821  detector = inputs['ccdExposure'].getDetector()
822  if self.doLinearize(detector) is True:
823  if 'linearizer' not in inputs:
824  linearityName = detector.getAmplifiers()[0].getLinearityType()
825  inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
826 
827  if self.config.doDefect is True:
828  if "defects" in inputs and inputs['defects'] is not None:
829  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
830  # masking expects a list of defects defined by their bounding box
831  if not isinstance(inputs["defects"], Defects):
832  inputs["defects"] = Defects.fromTable(inputs["defects"])
833 
834  # TODO: DM-22776 add retrieval for brighter-fatter kernel for Gen3
835  # if we can get HSC to use a new kernel, or just translate the old
836  # one to the new format we could drop the whole new/old style support
837 
838  # Broken: DM-17169
839  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
840  # fixed for non-HSC cameras in the future.
841  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
842  # if self.config.doCrosstalk else None)
843 
844  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
845  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
846  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
847  expId=expId,
848  assembler=self.assembleCcd
849  if self.config.doAssembleIsrExposures else None)
850  else:
851  inputs['fringes'] = pipeBase.Struct(fringes=None)
852 
853  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
854  if 'strayLightData' not in inputs:
855  inputs['strayLightData'] = None
856 
857  outputs = self.run(**inputs)
858  butlerQC.put(outputs, outputRefs)
859 
860  def readIsrData(self, dataRef, rawExposure):
861  """!Retrieve necessary frames for instrument signature removal.
862 
863  Pre-fetching all required ISR data products limits the IO
864  required by the ISR. Any conflict between the calibration data
865  available and that needed for ISR is also detected prior to
866  doing processing, allowing it to fail quickly.
867 
868  Parameters
869  ----------
870  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
871  Butler reference of the detector data to be processed
872  rawExposure : `afw.image.Exposure`
873  The raw exposure that will later be corrected with the
874  retrieved calibration data; should not be modified in this
875  method.
876 
877  Returns
878  -------
879  result : `lsst.pipe.base.Struct`
880  Result struct with components (which may be `None`):
881  - ``bias``: bias calibration frame (`afw.image.Exposure`)
882  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
883  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
884  - ``dark``: dark calibration frame (`afw.image.Exposure`)
885  - ``flat``: flat calibration frame (`afw.image.Exposure`)
886  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
887  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
888  - ``fringes``: `lsst.pipe.base.Struct` with components:
889  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
890  - ``seed``: random seed derived from the ccdExposureId for random
891  number generator (`uint32`).
892  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
893  A ``TransmissionCurve`` that represents the throughput of the optics,
894  to be evaluated in focal-plane coordinates.
895  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
896  A ``TransmissionCurve`` that represents the throughput of the filter
897  itself, to be evaluated in focal-plane coordinates.
898  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
899  A ``TransmissionCurve`` that represents the throughput of the sensor
900  itself, to be evaluated in post-assembly trimmed detector coordinates.
901  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
902  A ``TransmissionCurve`` that represents the throughput of the
903  atmosphere, assumed to be spatially constant.
904  - ``strayLightData`` : `object`
905  An opaque object containing calibration information for
906  stray-light correction. If `None`, no correction will be
907  performed.
908  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
909 
910  Raises
911  ------
912  NotImplementedError :
913  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
914  """
915  try:
916  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
917  dateObs = dateObs.toPython().isoformat()
918  except RuntimeError:
919  self.log.warn("Unable to identify dateObs for rawExposure.")
920  dateObs = None
921 
922  ccd = rawExposure.getDetector()
923  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
924  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
925  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
926  if self.config.doBias else None)
927  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
928  linearizer = (dataRef.get("linearizer", immediate=True)
929  if self.doLinearize(ccd) else None)
930  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
931  if self.config.doCrosstalk else None)
932  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
933  if self.config.doDark else None)
934  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
935  dateObs=dateObs)
936  if self.config.doFlat else None)
937 
938  brighterFatterKernel = None
939  brighterFatterGains = None
940  if self.config.doBrighterFatter is True:
941  try:
942  # Use the new-style cp_pipe version of the kernel if it exists
943  # If using a new-style kernel, always use the self-consistent
944  # gains, i.e. the ones inside the kernel object itself
945  brighterFatterKernel = dataRef.get("brighterFatterKernel")
946  brighterFatterGains = brighterFatterKernel.gain
947  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
948  except NoResults:
949  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
950  brighterFatterKernel = dataRef.get("bfKernel")
951  self.log.info("Old style bright-fatter kernel (np.array) loaded")
952  except NoResults:
953  brighterFatterKernel = None
954  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
955  # If the kernel is not an ndarray, it's the cp_pipe version
956  # so extract the kernel for this detector, or raise an error
957  if self.config.brighterFatterLevel == 'DETECTOR':
958  if brighterFatterKernel.detectorKernel:
959  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
960  elif brighterFatterKernel.detectorKernelFromAmpKernels:
961  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
962  else:
963  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
964  else:
965  # TODO DM-15631 for implementing this
966  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
967 
968  defectList = (dataRef.get("defects")
969  if self.config.doDefect else None)
970  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
971  if self.config.doAssembleIsrExposures else None)
972  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
973  else pipeBase.Struct(fringes=None))
974 
975  if self.config.doAttachTransmissionCurve:
976  opticsTransmission = (dataRef.get("transmission_optics")
977  if self.config.doUseOpticsTransmission else None)
978  filterTransmission = (dataRef.get("transmission_filter")
979  if self.config.doUseFilterTransmission else None)
980  sensorTransmission = (dataRef.get("transmission_sensor")
981  if self.config.doUseSensorTransmission else None)
982  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
983  if self.config.doUseAtmosphereTransmission else None)
984  else:
985  opticsTransmission = None
986  filterTransmission = None
987  sensorTransmission = None
988  atmosphereTransmission = None
989 
990  if self.config.doStrayLight:
991  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
992  else:
993  strayLightData = None
994 
995  illumMaskedImage = (self.getIsrExposure(dataRef,
996  self.config.illuminationCorrectionDataProductName).getMaskedImage()
997  if (self.config.doIlluminationCorrection and
998  filterName in self.config.illumFilters)
999  else None)
1000 
1001  # Struct should include only kwargs to run()
1002  return pipeBase.Struct(bias=biasExposure,
1003  linearizer=linearizer,
1004  crosstalkSources=crosstalkSources,
1005  dark=darkExposure,
1006  flat=flatExposure,
1007  bfKernel=brighterFatterKernel,
1008  bfGains=brighterFatterGains,
1009  defects=defectList,
1010  fringes=fringeStruct,
1011  opticsTransmission=opticsTransmission,
1012  filterTransmission=filterTransmission,
1013  sensorTransmission=sensorTransmission,
1014  atmosphereTransmission=atmosphereTransmission,
1015  strayLightData=strayLightData,
1016  illumMaskedImage=illumMaskedImage
1017  )
1018 
1019  @pipeBase.timeMethod
1020  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1021  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1022  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1023  sensorTransmission=None, atmosphereTransmission=None,
1024  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1025  isGen3=False,
1026  ):
1027  """!Perform instrument signature removal on an exposure.
1028 
1029  Steps included in the ISR processing, in order performed, are:
1030  - saturation and suspect pixel masking
1031  - overscan subtraction
1032  - CCD assembly of individual amplifiers
1033  - bias subtraction
1034  - variance image construction
1035  - linearization of non-linear response
1036  - crosstalk masking
1037  - brighter-fatter correction
1038  - dark subtraction
1039  - fringe correction
1040  - stray light subtraction
1041  - flat correction
1042  - masking of known defects and camera specific features
1043  - vignette calculation
1044  - appending transmission curve and distortion model
1045 
1046  Parameters
1047  ----------
1048  ccdExposure : `lsst.afw.image.Exposure`
1049  The raw exposure that is to be run through ISR. The
1050  exposure is modified by this method.
1051  camera : `lsst.afw.cameraGeom.Camera`, optional
1052  The camera geometry for this exposure. Used to select the
1053  distortion model appropriate for this data.
1054  bias : `lsst.afw.image.Exposure`, optional
1055  Bias calibration frame.
1056  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1057  Functor for linearization.
1058  crosstalkSources : `list`, optional
1059  List of possible crosstalk sources.
1060  dark : `lsst.afw.image.Exposure`, optional
1061  Dark calibration frame.
1062  flat : `lsst.afw.image.Exposure`, optional
1063  Flat calibration frame.
1064  bfKernel : `numpy.ndarray`, optional
1065  Brighter-fatter kernel.
1066  bfGains : `dict` of `float`, optional
1067  Gains used to override the detector's nominal gains for the
1068  brighter-fatter correction. A dict keyed by amplifier name for
1069  the detector in question.
1070  defects : `lsst.meas.algorithms.Defects`, optional
1071  List of defects.
1072  fringes : `lsst.pipe.base.Struct`, optional
1073  Struct containing the fringe correction data, with
1074  elements:
1075  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1076  - ``seed``: random seed derived from the ccdExposureId for random
1077  number generator (`uint32`)
1078  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1079  A ``TransmissionCurve`` that represents the throughput of the optics,
1080  to be evaluated in focal-plane coordinates.
1081  filterTransmission : `lsst.afw.image.TransmissionCurve`
1082  A ``TransmissionCurve`` that represents the throughput of the filter
1083  itself, to be evaluated in focal-plane coordinates.
1084  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1085  A ``TransmissionCurve`` that represents the throughput of the sensor
1086  itself, to be evaluated in post-assembly trimmed detector coordinates.
1087  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1088  A ``TransmissionCurve`` that represents the throughput of the
1089  atmosphere, assumed to be spatially constant.
1090  detectorNum : `int`, optional
1091  The integer number for the detector to process.
1092  isGen3 : bool, optional
1093  Flag this call to run() as using the Gen3 butler environment.
1094  strayLightData : `object`, optional
1095  Opaque object containing calibration information for stray-light
1096  correction. If `None`, no correction will be performed.
1097  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1098  Illumination correction image.
1099 
1100  Returns
1101  -------
1102  result : `lsst.pipe.base.Struct`
1103  Result struct with component:
1104  - ``exposure`` : `afw.image.Exposure`
1105  The fully ISR corrected exposure.
1106  - ``outputExposure`` : `afw.image.Exposure`
1107  An alias for `exposure`
1108  - ``ossThumb`` : `numpy.ndarray`
1109  Thumbnail image of the exposure after overscan subtraction.
1110  - ``flattenedThumb`` : `numpy.ndarray`
1111  Thumbnail image of the exposure after flat-field correction.
1112 
1113  Raises
1114  ------
1115  RuntimeError
1116  Raised if a configuration option is set to True, but the
1117  required calibration data has not been specified.
1118 
1119  Notes
1120  -----
1121  The current processed exposure can be viewed by setting the
1122  appropriate lsstDebug entries in the `debug.display`
1123  dictionary. The names of these entries correspond to some of
1124  the IsrTaskConfig Boolean options, with the value denoting the
1125  frame to use. The exposure is shown inside the matching
1126  option check and after the processing of that step has
1127  finished. The steps with debug points are:
1128 
1129  doAssembleCcd
1130  doBias
1131  doCrosstalk
1132  doBrighterFatter
1133  doDark
1134  doFringe
1135  doStrayLight
1136  doFlat
1137 
1138  In addition, setting the "postISRCCD" entry displays the
1139  exposure after all ISR processing has finished.
1140 
1141  """
1142 
1143  if isGen3 is True:
1144  # Gen3 currently cannot automatically do configuration overrides.
1145  # DM-15257 looks to discuss this issue.
1146  # Configure input exposures;
1147  if detectorNum is None:
1148  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1149 
1150  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1151  bias = self.ensureExposure(bias, camera, detectorNum)
1152  dark = self.ensureExposure(dark, camera, detectorNum)
1153  flat = self.ensureExposure(flat, camera, detectorNum)
1154  else:
1155  if isinstance(ccdExposure, ButlerDataRef):
1156  return self.runDataRef(ccdExposure)
1157 
1158  ccd = ccdExposure.getDetector()
1159  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1160 
1161  if not ccd:
1162  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1163  ccd = [FakeAmp(ccdExposure, self.config)]
1164 
1165  # Validate Input
1166  if self.config.doBias and bias is None:
1167  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1168  if self.doLinearize(ccd) and linearizer is None:
1169  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1170  if self.config.doBrighterFatter and bfKernel is None:
1171  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1172  if self.config.doDark and dark is None:
1173  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1174  if self.config.doFlat and flat is None:
1175  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1176  if self.config.doDefect and defects is None:
1177  raise RuntimeError("Must supply defects if config.doDefect=True.")
1178  if (self.config.doFringe and filterName in self.fringe.config.filters and
1179  fringes.fringes is None):
1180  # The `fringes` object needs to be a pipeBase.Struct, as
1181  # we use it as a `dict` for the parameters of
1182  # `FringeTask.run()`. The `fringes.fringes` `list` may
1183  # not be `None` if `doFringe=True`. Otherwise, raise.
1184  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1185  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1186  illumMaskedImage is None):
1187  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1188 
1189  # Begin ISR processing.
1190  if self.config.doConvertIntToFloat:
1191  self.log.info("Converting exposure to floating point values.")
1192  ccdExposure = self.convertIntToFloat(ccdExposure)
1193 
1194  # Amplifier level processing.
1195  overscans = []
1196  for amp in ccd:
1197  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1198  if ccdExposure.getBBox().contains(amp.getBBox()):
1199  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1200  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1201 
1202  if self.config.doOverscan and not badAmp:
1203  # Overscan correction on amp-by-amp basis.
1204  overscanResults = self.overscanCorrection(ccdExposure, amp)
1205  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1206  if overscanResults is not None and \
1207  self.config.qa is not None and self.config.qa.saveStats is True:
1208  if isinstance(overscanResults.overscanFit, float):
1209  qaMedian = overscanResults.overscanFit
1210  qaStdev = float("NaN")
1211  else:
1212  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1213  afwMath.MEDIAN | afwMath.STDEVCLIP)
1214  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1215  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1216 
1217  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1218  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1219  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1220  amp.getName(), qaMedian, qaStdev)
1221  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1222  else:
1223  if badAmp:
1224  self.log.warn("Amplifier %s is bad.", amp.getName())
1225  overscanResults = None
1226 
1227  overscans.append(overscanResults if overscanResults is not None else None)
1228  else:
1229  self.log.info("Skipped OSCAN for %s.", amp.getName())
1230 
1231  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1232  self.log.info("Applying crosstalk correction.")
1233  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1234  self.debugView(ccdExposure, "doCrosstalk")
1235 
1236  if self.config.doAssembleCcd:
1237  self.log.info("Assembling CCD from amplifiers.")
1238  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1239 
1240  if self.config.expectWcs and not ccdExposure.getWcs():
1241  self.log.warn("No WCS found in input exposure.")
1242  self.debugView(ccdExposure, "doAssembleCcd")
1243 
1244  ossThumb = None
1245  if self.config.qa.doThumbnailOss:
1246  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1247 
1248  if self.config.doBias:
1249  self.log.info("Applying bias correction.")
1250  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1251  trimToFit=self.config.doTrimToMatchCalib)
1252  self.debugView(ccdExposure, "doBias")
1253 
1254  if self.config.doVariance:
1255  for amp, overscanResults in zip(ccd, overscans):
1256  if ccdExposure.getBBox().contains(amp.getBBox()):
1257  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1258  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1259  if overscanResults is not None:
1260  self.updateVariance(ampExposure, amp,
1261  overscanImage=overscanResults.overscanImage)
1262  else:
1263  self.updateVariance(ampExposure, amp,
1264  overscanImage=None)
1265  if self.config.qa is not None and self.config.qa.saveStats is True:
1266  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1267  afwMath.MEDIAN | afwMath.STDEVCLIP)
1268  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1269  qaStats.getValue(afwMath.MEDIAN))
1270  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1271  qaStats.getValue(afwMath.STDEVCLIP))
1272  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1273  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1274  qaStats.getValue(afwMath.STDEVCLIP))
1275 
1276  if self.doLinearize(ccd):
1277  self.log.info("Applying linearizer.")
1278  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1279 
1280  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1281  self.log.info("Applying crosstalk correction.")
1282  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1283  self.debugView(ccdExposure, "doCrosstalk")
1284 
1285  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1286  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1287  if self.config.doDefect:
1288  self.log.info("Masking defects.")
1289  self.maskDefect(ccdExposure, defects)
1290 
1291  if self.config.numEdgeSuspect > 0:
1292  self.log.info("Masking edges as SUSPECT.")
1293  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1294  maskPlane="SUSPECT")
1295 
1296  if self.config.doNanMasking:
1297  self.log.info("Masking NAN value pixels.")
1298  self.maskNan(ccdExposure)
1299 
1300  if self.config.doWidenSaturationTrails:
1301  self.log.info("Widening saturation trails.")
1302  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1303 
1304  if self.config.doCameraSpecificMasking:
1305  self.log.info("Masking regions for camera specific reasons.")
1306  self.masking.run(ccdExposure)
1307 
1308  if self.config.doBrighterFatter:
1309  # We need to apply flats and darks before we can interpolate, and we
1310  # need to interpolate before we do B-F, but we do B-F without the
1311  # flats and darks applied so we can work in units of electrons or holes.
1312  # This context manager applies and then removes the darks and flats.
1313  #
1314  # We also do not want to interpolate values here, so operate on temporary
1315  # images so we can apply only the BF-correction and roll back the
1316  # interpolation.
1317  interpExp = ccdExposure.clone()
1318  with self.flatContext(interpExp, flat, dark):
1319  isrFunctions.interpolateFromMask(
1320  maskedImage=interpExp.getMaskedImage(),
1321  fwhm=self.config.fwhm,
1322  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1323  maskNameList=self.config.maskListToInterpolate
1324  )
1325  bfExp = interpExp.clone()
1326 
1327  self.log.info("Applying brighter fatter correction.")
1328  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1329  self.config.brighterFatterMaxIter,
1330  self.config.brighterFatterThreshold,
1331  self.config.brighterFatterApplyGain,
1332  bfGains)
1333  if bfResults[1] == self.config.brighterFatterMaxIter:
1334  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1335  bfResults[0])
1336  else:
1337  self.log.info("Finished brighter fatter correction in %d iterations.",
1338  bfResults[1])
1339  image = ccdExposure.getMaskedImage().getImage()
1340  bfCorr = bfExp.getMaskedImage().getImage()
1341  bfCorr -= interpExp.getMaskedImage().getImage()
1342  image += bfCorr
1343 
1344  # Applying the brighter-fatter correction applies a
1345  # convolution to the science image. At the edges this
1346  # convolution may not have sufficient valid pixels to
1347  # produce a valid correction. Mark pixels within the size
1348  # of the brighter-fatter kernel as EDGE to warn of this
1349  # fact.
1350  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1351  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1352  maskPlane="EDGE")
1353 
1354  if self.config.brighterFatterMaskGrowSize > 0:
1355  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1356  for maskPlane in self.config.maskListToInterpolate:
1357  isrFunctions.growMasks(ccdExposure.getMask(),
1358  radius=self.config.brighterFatterMaskGrowSize,
1359  maskNameList=maskPlane,
1360  maskValue=maskPlane)
1361 
1362  self.debugView(ccdExposure, "doBrighterFatter")
1363 
1364  if self.config.doDark:
1365  self.log.info("Applying dark correction.")
1366  self.darkCorrection(ccdExposure, dark)
1367  self.debugView(ccdExposure, "doDark")
1368 
1369  if self.config.doFringe and not self.config.fringeAfterFlat:
1370  self.log.info("Applying fringe correction before flat.")
1371  self.fringe.run(ccdExposure, **fringes.getDict())
1372  self.debugView(ccdExposure, "doFringe")
1373 
1374  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1375  self.log.info("Checking strayLight correction.")
1376  self.strayLight.run(ccdExposure, strayLightData)
1377  self.debugView(ccdExposure, "doStrayLight")
1378 
1379  if self.config.doFlat:
1380  self.log.info("Applying flat correction.")
1381  self.flatCorrection(ccdExposure, flat)
1382  self.debugView(ccdExposure, "doFlat")
1383 
1384  if self.config.doApplyGains:
1385  self.log.info("Applying gain correction instead of flat.")
1386  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1387 
1388  if self.config.doFringe and self.config.fringeAfterFlat:
1389  self.log.info("Applying fringe correction after flat.")
1390  self.fringe.run(ccdExposure, **fringes.getDict())
1391 
1392  if self.config.doVignette:
1393  self.log.info("Constructing Vignette polygon.")
1394  self.vignettePolygon = self.vignette.run(ccdExposure)
1395 
1396  if self.config.vignette.doWriteVignettePolygon:
1397  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1398 
1399  if self.config.doAttachTransmissionCurve:
1400  self.log.info("Adding transmission curves.")
1401  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1402  filterTransmission=filterTransmission,
1403  sensorTransmission=sensorTransmission,
1404  atmosphereTransmission=atmosphereTransmission)
1405 
1406  flattenedThumb = None
1407  if self.config.qa.doThumbnailFlattened:
1408  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1409 
1410  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1411  self.log.info("Performing illumination correction.")
1412  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1413  illumMaskedImage, illumScale=self.config.illumScale,
1414  trimToFit=self.config.doTrimToMatchCalib)
1415 
1416  preInterpExp = None
1417  if self.config.doSaveInterpPixels:
1418  preInterpExp = ccdExposure.clone()
1419 
1420  # Reset and interpolate bad pixels.
1421  #
1422  # Large contiguous bad regions (which should have the BAD mask
1423  # bit set) should have their values set to the image median.
1424  # This group should include defects and bad amplifiers. As the
1425  # area covered by these defects are large, there's little
1426  # reason to expect that interpolation would provide a more
1427  # useful value.
1428  #
1429  # Smaller defects can be safely interpolated after the larger
1430  # regions have had their pixel values reset. This ensures
1431  # that the remaining defects adjacent to bad amplifiers (as an
1432  # example) do not attempt to interpolate extreme values.
1433  if self.config.doSetBadRegions:
1434  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1435  if badPixelCount > 0:
1436  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1437 
1438  if self.config.doInterpolate:
1439  self.log.info("Interpolating masked pixels.")
1440  isrFunctions.interpolateFromMask(
1441  maskedImage=ccdExposure.getMaskedImage(),
1442  fwhm=self.config.fwhm,
1443  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1444  maskNameList=list(self.config.maskListToInterpolate)
1445  )
1446 
1447  self.roughZeroPoint(ccdExposure)
1448 
1449  if self.config.doMeasureBackground:
1450  self.log.info("Measuring background level.")
1451  self.measureBackground(ccdExposure, self.config.qa)
1452 
1453  if self.config.qa is not None and self.config.qa.saveStats is True:
1454  for amp in ccd:
1455  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1456  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1457  afwMath.MEDIAN | afwMath.STDEVCLIP)
1458  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1459  qaStats.getValue(afwMath.MEDIAN))
1460  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1461  qaStats.getValue(afwMath.STDEVCLIP))
1462  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1463  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1464  qaStats.getValue(afwMath.STDEVCLIP))
1465 
1466  self.debugView(ccdExposure, "postISRCCD")
1467 
1468  return pipeBase.Struct(
1469  exposure=ccdExposure,
1470  ossThumb=ossThumb,
1471  flattenedThumb=flattenedThumb,
1472 
1473  preInterpolatedExposure=preInterpExp,
1474  outputExposure=ccdExposure,
1475  outputOssThumbnail=ossThumb,
1476  outputFlattenedThumbnail=flattenedThumb,
1477  )
1478 
1479  @pipeBase.timeMethod
1480  def runDataRef(self, sensorRef):
1481  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1482 
1483  This method contains the `CmdLineTask` interface to the ISR
1484  processing. All IO is handled here, freeing the `run()` method
1485  to manage only pixel-level calculations. The steps performed
1486  are:
1487  - Read in necessary detrending/isr/calibration data.
1488  - Process raw exposure in `run()`.
1489  - Persist the ISR-corrected exposure as "postISRCCD" if
1490  config.doWrite=True.
1491 
1492  Parameters
1493  ----------
1494  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1495  DataRef of the detector data to be processed
1496 
1497  Returns
1498  -------
1499  result : `lsst.pipe.base.Struct`
1500  Result struct with component:
1501  - ``exposure`` : `afw.image.Exposure`
1502  The fully ISR corrected exposure.
1503 
1504  Raises
1505  ------
1506  RuntimeError
1507  Raised if a configuration option is set to True, but the
1508  required calibration data does not exist.
1509 
1510  """
1511  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1512 
1513  ccdExposure = sensorRef.get(self.config.datasetType)
1514 
1515  camera = sensorRef.get("camera")
1516  isrData = self.readIsrData(sensorRef, ccdExposure)
1517 
1518  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1519 
1520  if self.config.doWrite:
1521  sensorRef.put(result.exposure, "postISRCCD")
1522  if result.preInterpolatedExposure is not None:
1523  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1524  if result.ossThumb is not None:
1525  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1526  if result.flattenedThumb is not None:
1527  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1528 
1529  return result
1530 
1531  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1532  """!Retrieve a calibration dataset for removing instrument signature.
1533 
1534  Parameters
1535  ----------
1536 
1537  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1538  DataRef of the detector data to find calibration datasets
1539  for.
1540  datasetType : `str`
1541  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1542  dateObs : `str`, optional
1543  Date of the observation. Used to correct butler failures
1544  when using fallback filters.
1545  immediate : `Bool`
1546  If True, disable butler proxies to enable error handling
1547  within this routine.
1548 
1549  Returns
1550  -------
1551  exposure : `lsst.afw.image.Exposure`
1552  Requested calibration frame.
1553 
1554  Raises
1555  ------
1556  RuntimeError
1557  Raised if no matching calibration frame can be found.
1558  """
1559  try:
1560  exp = dataRef.get(datasetType, immediate=immediate)
1561  except Exception as exc1:
1562  if not self.config.fallbackFilterName:
1563  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1564  try:
1565  if self.config.useFallbackDate and dateObs:
1566  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1567  dateObs=dateObs, immediate=immediate)
1568  else:
1569  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1570  except Exception as exc2:
1571  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1572  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1573  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1574 
1575  if self.config.doAssembleIsrExposures:
1576  exp = self.assembleCcd.assembleCcd(exp)
1577  return exp
1578 
1579  def ensureExposure(self, inputExp, camera, detectorNum):
1580  """Ensure that the data returned by Butler is a fully constructed exposure.
1581 
1582  ISR requires exposure-level image data for historical reasons, so if we did
1583  not recieve that from Butler, construct it from what we have, modifying the
1584  input in place.
1585 
1586  Parameters
1587  ----------
1588  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1589  `lsst.afw.image.ImageF`
1590  The input data structure obtained from Butler.
1591  camera : `lsst.afw.cameraGeom.camera`
1592  The camera associated with the image. Used to find the appropriate
1593  detector.
1594  detectorNum : `int`
1595  The detector this exposure should match.
1596 
1597  Returns
1598  -------
1599  inputExp : `lsst.afw.image.Exposure`
1600  The re-constructed exposure, with appropriate detector parameters.
1601 
1602  Raises
1603  ------
1604  TypeError
1605  Raised if the input data cannot be used to construct an exposure.
1606  """
1607  if isinstance(inputExp, afwImage.DecoratedImageU):
1608  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1609  elif isinstance(inputExp, afwImage.ImageF):
1610  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1611  elif isinstance(inputExp, afwImage.MaskedImageF):
1612  inputExp = afwImage.makeExposure(inputExp)
1613  elif isinstance(inputExp, afwImage.Exposure):
1614  pass
1615  elif inputExp is None:
1616  # Assume this will be caught by the setup if it is a problem.
1617  return inputExp
1618  else:
1619  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1620  (type(inputExp), ))
1621 
1622  if inputExp.getDetector() is None:
1623  inputExp.setDetector(camera[detectorNum])
1624 
1625  return inputExp
1626 
1627  def convertIntToFloat(self, exposure):
1628  """Convert exposure image from uint16 to float.
1629 
1630  If the exposure does not need to be converted, the input is
1631  immediately returned. For exposures that are converted to use
1632  floating point pixels, the variance is set to unity and the
1633  mask to zero.
1634 
1635  Parameters
1636  ----------
1637  exposure : `lsst.afw.image.Exposure`
1638  The raw exposure to be converted.
1639 
1640  Returns
1641  -------
1642  newexposure : `lsst.afw.image.Exposure`
1643  The input ``exposure``, converted to floating point pixels.
1644 
1645  Raises
1646  ------
1647  RuntimeError
1648  Raised if the exposure type cannot be converted to float.
1649 
1650  """
1651  if isinstance(exposure, afwImage.ExposureF):
1652  # Nothing to be done
1653  self.log.debug("Exposure already of type float.")
1654  return exposure
1655  if not hasattr(exposure, "convertF"):
1656  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1657 
1658  newexposure = exposure.convertF()
1659  newexposure.variance[:] = 1
1660  newexposure.mask[:] = 0x0
1661 
1662  return newexposure
1663 
1664  def maskAmplifier(self, ccdExposure, amp, defects):
1665  """Identify bad amplifiers, saturated and suspect pixels.
1666 
1667  Parameters
1668  ----------
1669  ccdExposure : `lsst.afw.image.Exposure`
1670  Input exposure to be masked.
1671  amp : `lsst.afw.table.AmpInfoCatalog`
1672  Catalog of parameters defining the amplifier on this
1673  exposure to mask.
1674  defects : `lsst.meas.algorithms.Defects`
1675  List of defects. Used to determine if the entire
1676  amplifier is bad.
1677 
1678  Returns
1679  -------
1680  badAmp : `Bool`
1681  If this is true, the entire amplifier area is covered by
1682  defects and unusable.
1683 
1684  """
1685  maskedImage = ccdExposure.getMaskedImage()
1686 
1687  badAmp = False
1688 
1689  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1690  # comparison with current defects definition.
1691  if defects is not None:
1692  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1693 
1694  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1695  # association with pixels in current ccdExposure).
1696  if badAmp:
1697  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1698  afwImage.PARENT)
1699  maskView = dataView.getMask()
1700  maskView |= maskView.getPlaneBitMask("BAD")
1701  del maskView
1702  return badAmp
1703 
1704  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1705  # Saturation and suspect pixels can be masked now, though.
1706  limits = dict()
1707  if self.config.doSaturation and not badAmp:
1708  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1709  if self.config.doSuspect and not badAmp:
1710  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1711  if math.isfinite(self.config.saturation):
1712  limits.update({self.config.saturatedMaskName: self.config.saturation})
1713 
1714  for maskName, maskThreshold in limits.items():
1715  if not math.isnan(maskThreshold):
1716  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1717  isrFunctions.makeThresholdMask(
1718  maskedImage=dataView,
1719  threshold=maskThreshold,
1720  growFootprints=0,
1721  maskName=maskName
1722  )
1723 
1724  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1725  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1726  afwImage.PARENT)
1727  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1728  self.config.suspectMaskName])
1729  if numpy.all(maskView.getArray() & maskVal > 0):
1730  badAmp = True
1731  maskView |= maskView.getPlaneBitMask("BAD")
1732 
1733  return badAmp
1734 
1735  def overscanCorrection(self, ccdExposure, amp):
1736  """Apply overscan correction in place.
1737 
1738  This method does initial pixel rejection of the overscan
1739  region. The overscan can also be optionally segmented to
1740  allow for discontinuous overscan responses to be fit
1741  separately. The actual overscan subtraction is performed by
1742  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1743  which is called here after the amplifier is preprocessed.
1744 
1745  Parameters
1746  ----------
1747  ccdExposure : `lsst.afw.image.Exposure`
1748  Exposure to have overscan correction performed.
1749  amp : `lsst.afw.table.AmpInfoCatalog`
1750  The amplifier to consider while correcting the overscan.
1751 
1752  Returns
1753  -------
1754  overscanResults : `lsst.pipe.base.Struct`
1755  Result struct with components:
1756  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1757  Value or fit subtracted from the amplifier image data.
1758  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1759  Value or fit subtracted from the overscan image data.
1760  - ``overscanImage`` : `lsst.afw.image.Image`
1761  Image of the overscan region with the overscan
1762  correction applied. This quantity is used to estimate
1763  the amplifier read noise empirically.
1764 
1765  Raises
1766  ------
1767  RuntimeError
1768  Raised if the ``amp`` does not contain raw pixel information.
1769 
1770  See Also
1771  --------
1772  lsst.ip.isr.isrFunctions.overscanCorrection
1773  """
1774  if not amp.getHasRawInfo():
1775  raise RuntimeError("This method must be executed on an amp with raw information.")
1776 
1777  if amp.getRawHorizontalOverscanBBox().isEmpty():
1778  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1779  return None
1780 
1781  statControl = afwMath.StatisticsControl()
1782  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1783 
1784  # Determine the bounding boxes
1785  dataBBox = amp.getRawDataBBox()
1786  oscanBBox = amp.getRawHorizontalOverscanBBox()
1787  dx0 = 0
1788  dx1 = 0
1789 
1790  prescanBBox = amp.getRawPrescanBBox()
1791  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1792  dx0 += self.config.overscanNumLeadingColumnsToSkip
1793  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1794  else:
1795  dx0 += self.config.overscanNumTrailingColumnsToSkip
1796  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1797 
1798  # Determine if we need to work on subregions of the amplifier and overscan.
1799  imageBBoxes = []
1800  overscanBBoxes = []
1801 
1802  if ((self.config.overscanBiasJump and
1803  self.config.overscanBiasJumpLocation) and
1804  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1805  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1806  self.config.overscanBiasJumpDevices)):
1807  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1808  yLower = self.config.overscanBiasJumpLocation
1809  yUpper = dataBBox.getHeight() - yLower
1810  else:
1811  yUpper = self.config.overscanBiasJumpLocation
1812  yLower = dataBBox.getHeight() - yUpper
1813 
1814  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1815  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1816  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1817  lsst.geom.Extent2I(dx0, 0),
1818  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1819  yLower)))
1820 
1821  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1822  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1823  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1824  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1825  yUpper)))
1826  else:
1827  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1828  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1829  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1830  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1831  oscanBBox.getHeight())))
1832 
1833  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1834  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1835  ampImage = ccdExposure.maskedImage[imageBBox]
1836  overscanImage = ccdExposure.maskedImage[overscanBBox]
1837 
1838  overscanArray = overscanImage.image.array
1839  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1840  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1841  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1842 
1843  statControl = afwMath.StatisticsControl()
1844  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1845 
1846  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1847  overscanImage=overscanImage,
1848  fitType=self.config.overscanFitType,
1849  order=self.config.overscanOrder,
1850  collapseRej=self.config.overscanNumSigmaClip,
1851  statControl=statControl,
1852  overscanIsInt=self.config.overscanIsInt
1853  )
1854 
1855  # Measure average overscan levels and record them in the metadata.
1856  levelStat = afwMath.MEDIAN
1857  sigmaStat = afwMath.STDEVCLIP
1858 
1859  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1860  self.config.qa.flatness.nIter)
1861  metadata = ccdExposure.getMetadata()
1862  ampNum = amp.getName()
1863  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1864  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1865  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1866  else:
1867  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1868  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1869  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1870 
1871  return overscanResults
1872 
1873  def updateVariance(self, ampExposure, amp, overscanImage=None):
1874  """Set the variance plane using the amplifier gain and read noise
1875 
1876  The read noise is calculated from the ``overscanImage`` if the
1877  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1878  the value from the amplifier data is used.
1879 
1880  Parameters
1881  ----------
1882  ampExposure : `lsst.afw.image.Exposure`
1883  Exposure to process.
1884  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1885  Amplifier detector data.
1886  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1887  Image of overscan, required only for empirical read noise.
1888 
1889  See also
1890  --------
1891  lsst.ip.isr.isrFunctions.updateVariance
1892  """
1893  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1894  gain = amp.getGain()
1895 
1896  if math.isnan(gain):
1897  gain = 1.0
1898  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1899  elif gain <= 0:
1900  patchedGain = 1.0
1901  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1902  amp.getName(), gain, patchedGain)
1903  gain = patchedGain
1904 
1905  if self.config.doEmpiricalReadNoise and overscanImage is None:
1906  self.log.info("Overscan is none for EmpiricalReadNoise.")
1907 
1908  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1909  stats = afwMath.StatisticsControl()
1910  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1911  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1912  self.log.info("Calculated empirical read noise for amp %s: %f.",
1913  amp.getName(), readNoise)
1914  else:
1915  readNoise = amp.getReadNoise()
1916 
1917  isrFunctions.updateVariance(
1918  maskedImage=ampExposure.getMaskedImage(),
1919  gain=gain,
1920  readNoise=readNoise,
1921  )
1922 
1923  def darkCorrection(self, exposure, darkExposure, invert=False):
1924  """!Apply dark correction in place.
1925 
1926  Parameters
1927  ----------
1928  exposure : `lsst.afw.image.Exposure`
1929  Exposure to process.
1930  darkExposure : `lsst.afw.image.Exposure`
1931  Dark exposure of the same size as ``exposure``.
1932  invert : `Bool`, optional
1933  If True, re-add the dark to an already corrected image.
1934 
1935  Raises
1936  ------
1937  RuntimeError
1938  Raised if either ``exposure`` or ``darkExposure`` do not
1939  have their dark time defined.
1940 
1941  See Also
1942  --------
1943  lsst.ip.isr.isrFunctions.darkCorrection
1944  """
1945  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1946  if math.isnan(expScale):
1947  raise RuntimeError("Exposure darktime is NAN.")
1948  if darkExposure.getInfo().getVisitInfo() is not None:
1949  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1950  else:
1951  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1952  # so getDarkTime() does not exist.
1953  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1954  darkScale = 1.0
1955 
1956  if math.isnan(darkScale):
1957  raise RuntimeError("Dark calib darktime is NAN.")
1958  isrFunctions.darkCorrection(
1959  maskedImage=exposure.getMaskedImage(),
1960  darkMaskedImage=darkExposure.getMaskedImage(),
1961  expScale=expScale,
1962  darkScale=darkScale,
1963  invert=invert,
1964  trimToFit=self.config.doTrimToMatchCalib
1965  )
1966 
1967  def doLinearize(self, detector):
1968  """!Check if linearization is needed for the detector cameraGeom.
1969 
1970  Checks config.doLinearize and the linearity type of the first
1971  amplifier.
1972 
1973  Parameters
1974  ----------
1975  detector : `lsst.afw.cameraGeom.Detector`
1976  Detector to get linearity type from.
1977 
1978  Returns
1979  -------
1980  doLinearize : `Bool`
1981  If True, linearization should be performed.
1982  """
1983  return self.config.doLinearize and \
1984  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
1985 
1986  def flatCorrection(self, exposure, flatExposure, invert=False):
1987  """!Apply flat correction in place.
1988 
1989  Parameters
1990  ----------
1991  exposure : `lsst.afw.image.Exposure`
1992  Exposure to process.
1993  flatExposure : `lsst.afw.image.Exposure`
1994  Flat exposure of the same size as ``exposure``.
1995  invert : `Bool`, optional
1996  If True, unflatten an already flattened image.
1997 
1998  See Also
1999  --------
2000  lsst.ip.isr.isrFunctions.flatCorrection
2001  """
2002  isrFunctions.flatCorrection(
2003  maskedImage=exposure.getMaskedImage(),
2004  flatMaskedImage=flatExposure.getMaskedImage(),
2005  scalingType=self.config.flatScalingType,
2006  userScale=self.config.flatUserScale,
2007  invert=invert,
2008  trimToFit=self.config.doTrimToMatchCalib
2009  )
2010 
2011  def saturationDetection(self, exposure, amp):
2012  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2013 
2014  Parameters
2015  ----------
2016  exposure : `lsst.afw.image.Exposure`
2017  Exposure to process. Only the amplifier DataSec is processed.
2018  amp : `lsst.afw.table.AmpInfoCatalog`
2019  Amplifier detector data.
2020 
2021  See Also
2022  --------
2023  lsst.ip.isr.isrFunctions.makeThresholdMask
2024  """
2025  if not math.isnan(amp.getSaturation()):
2026  maskedImage = exposure.getMaskedImage()
2027  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2028  isrFunctions.makeThresholdMask(
2029  maskedImage=dataView,
2030  threshold=amp.getSaturation(),
2031  growFootprints=0,
2032  maskName=self.config.saturatedMaskName,
2033  )
2034 
2035  def saturationInterpolation(self, exposure):
2036  """!Interpolate over saturated pixels, in place.
2037 
2038  This method should be called after `saturationDetection`, to
2039  ensure that the saturated pixels have been identified in the
2040  SAT mask. It should also be called after `assembleCcd`, since
2041  saturated regions may cross amplifier boundaries.
2042 
2043  Parameters
2044  ----------
2045  exposure : `lsst.afw.image.Exposure`
2046  Exposure to process.
2047 
2048  See Also
2049  --------
2050  lsst.ip.isr.isrTask.saturationDetection
2051  lsst.ip.isr.isrFunctions.interpolateFromMask
2052  """
2053  isrFunctions.interpolateFromMask(
2054  maskedImage=exposure.getMaskedImage(),
2055  fwhm=self.config.fwhm,
2056  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2057  maskNameList=list(self.config.saturatedMaskName),
2058  )
2059 
2060  def suspectDetection(self, exposure, amp):
2061  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2062 
2063  Parameters
2064  ----------
2065  exposure : `lsst.afw.image.Exposure`
2066  Exposure to process. Only the amplifier DataSec is processed.
2067  amp : `lsst.afw.table.AmpInfoCatalog`
2068  Amplifier detector data.
2069 
2070  See Also
2071  --------
2072  lsst.ip.isr.isrFunctions.makeThresholdMask
2073 
2074  Notes
2075  -----
2076  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2077  This is intended to indicate pixels that may be affected by unknown systematics;
2078  for example if non-linearity corrections above a certain level are unstable
2079  then that would be a useful value for suspectLevel. A value of `nan` indicates
2080  that no such level exists and no pixels are to be masked as suspicious.
2081  """
2082  suspectLevel = amp.getSuspectLevel()
2083  if math.isnan(suspectLevel):
2084  return
2085 
2086  maskedImage = exposure.getMaskedImage()
2087  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2088  isrFunctions.makeThresholdMask(
2089  maskedImage=dataView,
2090  threshold=suspectLevel,
2091  growFootprints=0,
2092  maskName=self.config.suspectMaskName,
2093  )
2094 
2095  def maskDefect(self, exposure, defectBaseList):
2096  """!Mask defects using mask plane "BAD", in place.
2097 
2098  Parameters
2099  ----------
2100  exposure : `lsst.afw.image.Exposure`
2101  Exposure to process.
2102  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2103  `lsst.afw.image.DefectBase`.
2104  List of defects to mask.
2105 
2106  Notes
2107  -----
2108  Call this after CCD assembly, since defects may cross amplifier boundaries.
2109  """
2110  maskedImage = exposure.getMaskedImage()
2111  if not isinstance(defectBaseList, Defects):
2112  # Promotes DefectBase to Defect
2113  defectList = Defects(defectBaseList)
2114  else:
2115  defectList = defectBaseList
2116  defectList.maskPixels(maskedImage, maskName="BAD")
2117 
2118  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2119  """!Mask edge pixels with applicable mask plane.
2120 
2121  Parameters
2122  ----------
2123  exposure : `lsst.afw.image.Exposure`
2124  Exposure to process.
2125  numEdgePixels : `int`, optional
2126  Number of edge pixels to mask.
2127  maskPlane : `str`, optional
2128  Mask plane name to use.
2129  """
2130  maskedImage = exposure.getMaskedImage()
2131  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2132 
2133  if numEdgePixels > 0:
2134  goodBBox = maskedImage.getBBox()
2135  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2136  goodBBox.grow(-numEdgePixels)
2137  # Mask pixels outside goodBBox
2138  SourceDetectionTask.setEdgeBits(
2139  maskedImage,
2140  goodBBox,
2141  maskBitMask
2142  )
2143 
2144  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2145  """Mask and interpolate defects using mask plane "BAD", in place.
2146 
2147  Parameters
2148  ----------
2149  exposure : `lsst.afw.image.Exposure`
2150  Exposure to process.
2151  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2152  `lsst.afw.image.DefectBase`.
2153  List of defects to mask and interpolate.
2154 
2155  See Also
2156  --------
2157  lsst.ip.isr.isrTask.maskDefect()
2158  """
2159  self.maskDefect(exposure, defectBaseList)
2160  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2161  maskPlane="SUSPECT")
2162  isrFunctions.interpolateFromMask(
2163  maskedImage=exposure.getMaskedImage(),
2164  fwhm=self.config.fwhm,
2165  growSaturatedFootprints=0,
2166  maskNameList=["BAD"],
2167  )
2168 
2169  def maskNan(self, exposure):
2170  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2171 
2172  Parameters
2173  ----------
2174  exposure : `lsst.afw.image.Exposure`
2175  Exposure to process.
2176 
2177  Notes
2178  -----
2179  We mask over all NaNs, including those that are masked with
2180  other bits (because those may or may not be interpolated over
2181  later, and we want to remove all NaNs). Despite this
2182  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2183  the historical name.
2184  """
2185  maskedImage = exposure.getMaskedImage()
2186 
2187  # Find and mask NaNs
2188  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2189  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2190  numNans = maskNans(maskedImage, maskVal)
2191  self.metadata.set("NUMNANS", numNans)
2192  if numNans > 0:
2193  self.log.warn("There were %d unmasked NaNs.", numNans)
2194 
2195  def maskAndInterpolateNan(self, exposure):
2196  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2197 
2198  Parameters
2199  ----------
2200  exposure : `lsst.afw.image.Exposure`
2201  Exposure to process.
2202 
2203  See Also
2204  --------
2205  lsst.ip.isr.isrTask.maskNan()
2206  """
2207  self.maskNan(exposure)
2208  isrFunctions.interpolateFromMask(
2209  maskedImage=exposure.getMaskedImage(),
2210  fwhm=self.config.fwhm,
2211  growSaturatedFootprints=0,
2212  maskNameList=["UNMASKEDNAN"],
2213  )
2214 
2215  def measureBackground(self, exposure, IsrQaConfig=None):
2216  """Measure the image background in subgrids, for quality control purposes.
2217 
2218  Parameters
2219  ----------
2220  exposure : `lsst.afw.image.Exposure`
2221  Exposure to process.
2222  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2223  Configuration object containing parameters on which background
2224  statistics and subgrids to use.
2225  """
2226  if IsrQaConfig is not None:
2227  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2228  IsrQaConfig.flatness.nIter)
2229  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2230  statsControl.setAndMask(maskVal)
2231  maskedImage = exposure.getMaskedImage()
2232  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2233  skyLevel = stats.getValue(afwMath.MEDIAN)
2234  skySigma = stats.getValue(afwMath.STDEVCLIP)
2235  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2236  metadata = exposure.getMetadata()
2237  metadata.set('SKYLEVEL', skyLevel)
2238  metadata.set('SKYSIGMA', skySigma)
2239 
2240  # calcluating flatlevel over the subgrids
2241  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2242  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2243  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2244  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2245  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2246  skyLevels = numpy.zeros((nX, nY))
2247 
2248  for j in range(nY):
2249  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2250  for i in range(nX):
2251  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2252 
2253  xLLC = xc - meshXHalf
2254  yLLC = yc - meshYHalf
2255  xURC = xc + meshXHalf - 1
2256  yURC = yc + meshYHalf - 1
2257 
2258  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2259  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2260 
2261  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2262 
2263  good = numpy.where(numpy.isfinite(skyLevels))
2264  skyMedian = numpy.median(skyLevels[good])
2265  flatness = (skyLevels[good] - skyMedian) / skyMedian
2266  flatness_rms = numpy.std(flatness)
2267  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2268 
2269  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2270  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2271  nX, nY, flatness_pp, flatness_rms)
2272 
2273  metadata.set('FLATNESS_PP', float(flatness_pp))
2274  metadata.set('FLATNESS_RMS', float(flatness_rms))
2275  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2276  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2277  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2278 
2279  def roughZeroPoint(self, exposure):
2280  """Set an approximate magnitude zero point for the exposure.
2281 
2282  Parameters
2283  ----------
2284  exposure : `lsst.afw.image.Exposure`
2285  Exposure to process.
2286  """
2287  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2288  if filterName in self.config.fluxMag0T1:
2289  fluxMag0 = self.config.fluxMag0T1[filterName]
2290  else:
2291  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2292  fluxMag0 = self.config.defaultFluxMag0T1
2293 
2294  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2295  if not expTime > 0: # handle NaN as well as <= 0
2296  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2297  return
2298 
2299  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2300  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2301 
2302  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2303  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2304 
2305  Parameters
2306  ----------
2307  ccdExposure : `lsst.afw.image.Exposure`
2308  Exposure to process.
2309  fpPolygon : `lsst.afw.geom.Polygon`
2310  Polygon in focal plane coordinates.
2311  """
2312  # Get ccd corners in focal plane coordinates
2313  ccd = ccdExposure.getDetector()
2314  fpCorners = ccd.getCorners(FOCAL_PLANE)
2315  ccdPolygon = Polygon(fpCorners)
2316 
2317  # Get intersection of ccd corners with fpPolygon
2318  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2319 
2320  # Transform back to pixel positions and build new polygon
2321  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2322  validPolygon = Polygon(ccdPoints)
2323  ccdExposure.getInfo().setValidPolygon(validPolygon)
2324 
2325  @contextmanager
2326  def flatContext(self, exp, flat, dark=None):
2327  """Context manager that applies and removes flats and darks,
2328  if the task is configured to apply them.
2329 
2330  Parameters
2331  ----------
2332  exp : `lsst.afw.image.Exposure`
2333  Exposure to process.
2334  flat : `lsst.afw.image.Exposure`
2335  Flat exposure the same size as ``exp``.
2336  dark : `lsst.afw.image.Exposure`, optional
2337  Dark exposure the same size as ``exp``.
2338 
2339  Yields
2340  ------
2341  exp : `lsst.afw.image.Exposure`
2342  The flat and dark corrected exposure.
2343  """
2344  if self.config.doDark and dark is not None:
2345  self.darkCorrection(exp, dark)
2346  if self.config.doFlat:
2347  self.flatCorrection(exp, flat)
2348  try:
2349  yield exp
2350  finally:
2351  if self.config.doFlat:
2352  self.flatCorrection(exp, flat, invert=True)
2353  if self.config.doDark and dark is not None:
2354  self.darkCorrection(exp, dark, invert=True)
2355 
2356  def debugView(self, exposure, stepname):
2357  """Utility function to examine ISR exposure at different stages.
2358 
2359  Parameters
2360  ----------
2361  exposure : `lsst.afw.image.Exposure`
2362  Exposure to view.
2363  stepname : `str`
2364  State of processing to view.
2365  """
2366  frame = getDebugFrame(self._display, stepname)
2367  if frame:
2368  display = getDisplay(frame)
2369  display.scale('asinh', 'zscale')
2370  display.mtv(exposure)
2371  prompt = "Press Enter to continue [c]... "
2372  while True:
2373  ans = input(prompt).lower()
2374  if ans in ("", "c",):
2375  break
2376 
2377 
2378 class FakeAmp(object):
2379  """A Detector-like object that supports returning gain and saturation level
2380 
2381  This is used when the input exposure does not have a detector.
2382 
2383  Parameters
2384  ----------
2385  exposure : `lsst.afw.image.Exposure`
2386  Exposure to generate a fake amplifier for.
2387  config : `lsst.ip.isr.isrTaskConfig`
2388  Configuration to apply to the fake amplifier.
2389  """
2390 
2391  def __init__(self, exposure, config):
2392  self._bbox = exposure.getBBox(afwImage.LOCAL)
2394  self._gain = config.gain
2395  self._readNoise = config.readNoise
2396  self._saturation = config.saturation
2397 
2398  def getBBox(self):
2399  return self._bbox
2400 
2401  def getRawBBox(self):
2402  return self._bbox
2403 
2404  def getHasRawInfo(self):
2405  return True # but see getRawHorizontalOverscanBBox()
2406 
2408  return self._RawHorizontalOverscanBBox
2409 
2410  def getGain(self):
2411  return self._gain
2412 
2413  def getReadNoise(self):
2414  return self._readNoise
2415 
2416  def getSaturation(self):
2417  return self._saturation
2418 
2419  def getSuspectLevel(self):
2420  return float("NaN")
2421 
2422 
2423 class RunIsrConfig(pexConfig.Config):
2424  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2425 
2426 
2427 class RunIsrTask(pipeBase.CmdLineTask):
2428  """Task to wrap the default IsrTask to allow it to be retargeted.
2429 
2430  The standard IsrTask can be called directly from a command line
2431  program, but doing so removes the ability of the task to be
2432  retargeted. As most cameras override some set of the IsrTask
2433  methods, this would remove those data-specific methods in the
2434  output post-ISR images. This wrapping class fixes the issue,
2435  allowing identical post-ISR images to be generated by both the
2436  processCcd and isrTask code.
2437  """
2438  ConfigClass = RunIsrConfig
2439  _DefaultName = "runIsr"
2440 
2441  def __init__(self, *args, **kwargs):
2442  super().__init__(*args, **kwargs)
2443  self.makeSubtask("isr")
2444 
2445  def runDataRef(self, dataRef):
2446  """
2447  Parameters
2448  ----------
2449  dataRef : `lsst.daf.persistence.ButlerDataRef`
2450  data reference of the detector data to be processed
2451 
2452  Returns
2453  -------
2454  result : `pipeBase.Struct`
2455  Result struct with component:
2456 
2457  - exposure : `lsst.afw.image.Exposure`
2458  Post-ISR processed exposure.
2459  """
2460  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1480
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2215
def debugView(self, exposure, stepname)
Definition: isrTask.py:2356
def __init__(self, kwargs)
Definition: isrTask.py:801
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1579
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:860
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2118
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:810
def runDataRef(self, dataRef)
Definition: isrTask.py:2445
def __init__(self, args, kwargs)
Definition: isrTask.py:2441
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2279
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2144
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2407
def maskNan(self, exposure)
Definition: isrTask.py:2169
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2095
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1735
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1627
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1986
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1026
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1923
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1967
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2302
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1664
def __init__(self, config=None)
Definition: isrTask.py:173
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2326
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1531
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1873
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2195
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2060
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2035
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:2011
def __init__(self, exposure, config)
Definition: isrTask.py:2391