lsst.ip.isr  19.0.0-14-g5673ca6+10
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.Input(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector", "exposure"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ExposureF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ExposureF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="ExposureF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  newBFKernel = cT.PrerequisiteInput(
112  name='brighterFatterKernel',
113  doc="Newer complete kernel + gain solutions.",
114  storageClass="BrighterFatterKernel",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  defects = cT.PrerequisiteInput(
118  name='defects',
119  doc="Input defect tables.",
120  storageClass="Defects",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  opticsTransmission = cT.PrerequisiteInput(
124  name="transmission_optics",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the optics.",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = cT.PrerequisiteInput(
130  name="transmission_filter",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the filter.",
133  dimensions=["instrument", "physical_filter", "calibration_label"],
134  )
135  sensorTransmission = cT.PrerequisiteInput(
136  name="transmission_sensor",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the sensor.",
139  dimensions=["instrument", "calibration_label", "detector"],
140  )
141  atmosphereTransmission = cT.PrerequisiteInput(
142  name="transmission_atmosphere",
143  storageClass="TransmissionCurve",
144  doc="Transmission curve due to the atmosphere.",
145  dimensions=["instrument"],
146  )
147  illumMaskedImage = cT.PrerequisiteInput(
148  name="illum",
149  doc="Input illumination correction.",
150  storageClass="MaskedImageF",
151  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
152  )
153 
154  outputExposure = cT.Output(
155  name='postISRCCD',
156  doc="Output ISR processed exposure.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "visit", "detector"],
159  )
160  preInterpExposure = cT.Output(
161  name='preInterpISRCCD',
162  doc="Output ISR processed exposure, with pixels left uninterpolated.",
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = cT.Output(
167  name="OssThumb",
168  doc="Output Overscan-subtracted thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "visit", "detector"],
171  )
172  outputFlattenedThumbnail = cT.Output(
173  name="FlattenedThumb",
174  doc="Output flat-corrected thumbnail image.",
175  storageClass="Thumbnail",
176  dimensions=["instrument", "visit", "detector"],
177  )
178 
179  def __init__(self, *, config=None):
180  super().__init__(config=config)
181 
182  if config.doBias is not True:
183  self.prerequisiteInputs.discard("bias")
184  if config.doLinearize is not True:
185  self.prerequisiteInputs.discard("linearizer")
186  if config.doCrosstalk is not True:
187  self.prerequisiteInputs.discard("crosstalkSources")
188  if config.doBrighterFatter is not True:
189  self.prerequisiteInputs.discard("bfKernel")
190  self.prerequisiteInputs.discard("newBFKernel")
191  if config.doDefect is not True:
192  self.prerequisiteInputs.discard("defects")
193  if config.doDark is not True:
194  self.prerequisiteInputs.discard("dark")
195  if config.doFlat is not True:
196  self.prerequisiteInputs.discard("flat")
197  if config.doAttachTransmissionCurve is not True:
198  self.prerequisiteInputs.discard("opticsTransmission")
199  self.prerequisiteInputs.discard("filterTransmission")
200  self.prerequisiteInputs.discard("sensorTransmission")
201  self.prerequisiteInputs.discard("atmosphereTransmission")
202  if config.doUseOpticsTransmission is not True:
203  self.prerequisiteInputs.discard("opticsTransmission")
204  if config.doUseFilterTransmission is not True:
205  self.prerequisiteInputs.discard("filterTransmission")
206  if config.doUseSensorTransmission is not True:
207  self.prerequisiteInputs.discard("sensorTransmission")
208  if config.doUseAtmosphereTransmission is not True:
209  self.prerequisiteInputs.discard("atmosphereTransmission")
210  if config.doIlluminationCorrection is not True:
211  self.prerequisiteInputs.discard("illumMaskedImage")
212 
213  if config.doWrite is not True:
214  self.outputs.discard("outputExposure")
215  self.outputs.discard("preInterpExposure")
216  self.outputs.discard("outputFlattenedThumbnail")
217  self.outputs.discard("outputOssThumbnail")
218  if config.doSaveInterpPixels is not True:
219  self.outputs.discard("preInterpExposure")
220  if config.qa.doThumbnailOss is not True:
221  self.outputs.discard("outputOssThumbnail")
222  if config.qa.doThumbnailFlattened is not True:
223  self.outputs.discard("outputFlattenedThumbnail")
224 
225 
226 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
227  pipelineConnections=IsrTaskConnections):
228  """Configuration parameters for IsrTask.
229 
230  Items are grouped in the order in which they are executed by the task.
231  """
232  datasetType = pexConfig.Field(
233  dtype=str,
234  doc="Dataset type for input data; users will typically leave this alone, "
235  "but camera-specific ISR tasks will override it",
236  default="raw",
237  )
238 
239  fallbackFilterName = pexConfig.Field(
240  dtype=str,
241  doc="Fallback default filter name for calibrations.",
242  optional=True
243  )
244  useFallbackDate = pexConfig.Field(
245  dtype=bool,
246  doc="Pass observation date when using fallback filter.",
247  default=False,
248  )
249  expectWcs = pexConfig.Field(
250  dtype=bool,
251  default=True,
252  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
253  )
254  fwhm = pexConfig.Field(
255  dtype=float,
256  doc="FWHM of PSF in arcseconds.",
257  default=1.0,
258  )
259  qa = pexConfig.ConfigField(
260  dtype=isrQa.IsrQaConfig,
261  doc="QA related configuration options.",
262  )
263 
264  # Image conversion configuration
265  doConvertIntToFloat = pexConfig.Field(
266  dtype=bool,
267  doc="Convert integer raw images to floating point values?",
268  default=True,
269  )
270 
271  # Saturated pixel handling.
272  doSaturation = pexConfig.Field(
273  dtype=bool,
274  doc="Mask saturated pixels? NB: this is totally independent of the"
275  " interpolation option - this is ONLY setting the bits in the mask."
276  " To have them interpolated make sure doSaturationInterpolation=True",
277  default=True,
278  )
279  saturatedMaskName = pexConfig.Field(
280  dtype=str,
281  doc="Name of mask plane to use in saturation detection and interpolation",
282  default="SAT",
283  )
284  saturation = pexConfig.Field(
285  dtype=float,
286  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287  default=float("NaN"),
288  )
289  growSaturationFootprintSize = pexConfig.Field(
290  dtype=int,
291  doc="Number of pixels by which to grow the saturation footprints",
292  default=1,
293  )
294 
295  # Suspect pixel handling.
296  doSuspect = pexConfig.Field(
297  dtype=bool,
298  doc="Mask suspect pixels?",
299  default=False,
300  )
301  suspectMaskName = pexConfig.Field(
302  dtype=str,
303  doc="Name of mask plane to use for suspect pixels",
304  default="SUSPECT",
305  )
306  numEdgeSuspect = pexConfig.Field(
307  dtype=int,
308  doc="Number of edge pixels to be flagged as untrustworthy.",
309  default=0,
310  )
311 
312  # Initial masking options.
313  doSetBadRegions = pexConfig.Field(
314  dtype=bool,
315  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
316  default=True,
317  )
318  badStatistic = pexConfig.ChoiceField(
319  dtype=str,
320  doc="How to estimate the average value for BAD regions.",
321  default='MEANCLIP',
322  allowed={
323  "MEANCLIP": "Correct using the (clipped) mean of good data",
324  "MEDIAN": "Correct using the median of the good data",
325  },
326  )
327 
328  # Overscan subtraction configuration.
329  doOverscan = pexConfig.Field(
330  dtype=bool,
331  doc="Do overscan subtraction?",
332  default=True,
333  )
334  overscanFitType = pexConfig.ChoiceField(
335  dtype=str,
336  doc="The method for fitting the overscan bias level.",
337  default='MEDIAN',
338  allowed={
339  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
340  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
341  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
342  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
343  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
344  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
345  "MEAN": "Correct using the mean of the overscan region",
346  "MEANCLIP": "Correct using a clipped mean of the overscan region",
347  "MEDIAN": "Correct using the median of the overscan region",
348  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
349  },
350  )
351  overscanOrder = pexConfig.Field(
352  dtype=int,
353  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
354  "or number of spline knots if overscan fit type is a spline."),
355  default=1,
356  )
357  overscanNumSigmaClip = pexConfig.Field(
358  dtype=float,
359  doc="Rejection threshold (sigma) for collapsing overscan before fit",
360  default=3.0,
361  )
362  overscanIsInt = pexConfig.Field(
363  dtype=bool,
364  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" +
365  " and overscan.FitType=MEDIAN_PER_ROW.",
366  default=True,
367  )
368  overscanNumLeadingColumnsToSkip = pexConfig.Field(
369  dtype=int,
370  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
371  default=0,
372  )
373  overscanNumTrailingColumnsToSkip = pexConfig.Field(
374  dtype=int,
375  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
376  default=0,
377  )
378  overscanMaxDev = pexConfig.Field(
379  dtype=float,
380  doc="Maximum deviation from the median for overscan",
381  default=1000.0, check=lambda x: x > 0
382  )
383  overscanBiasJump = pexConfig.Field(
384  dtype=bool,
385  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
386  default=False,
387  )
388  overscanBiasJumpKeyword = pexConfig.Field(
389  dtype=str,
390  doc="Header keyword containing information about devices.",
391  default="NO_SUCH_KEY",
392  )
393  overscanBiasJumpDevices = pexConfig.ListField(
394  dtype=str,
395  doc="List of devices that need piecewise overscan correction.",
396  default=(),
397  )
398  overscanBiasJumpLocation = pexConfig.Field(
399  dtype=int,
400  doc="Location of bias jump along y-axis.",
401  default=0,
402  )
403 
404  # Amplifier to CCD assembly configuration
405  doAssembleCcd = pexConfig.Field(
406  dtype=bool,
407  default=True,
408  doc="Assemble amp-level exposures into a ccd-level exposure?"
409  )
410  assembleCcd = pexConfig.ConfigurableField(
411  target=AssembleCcdTask,
412  doc="CCD assembly task",
413  )
414 
415  # General calibration configuration.
416  doAssembleIsrExposures = pexConfig.Field(
417  dtype=bool,
418  default=False,
419  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
420  )
421  doTrimToMatchCalib = pexConfig.Field(
422  dtype=bool,
423  default=False,
424  doc="Trim raw data to match calibration bounding boxes?"
425  )
426 
427  # Bias subtraction.
428  doBias = pexConfig.Field(
429  dtype=bool,
430  doc="Apply bias frame correction?",
431  default=True,
432  )
433  biasDataProductName = pexConfig.Field(
434  dtype=str,
435  doc="Name of the bias data product",
436  default="bias",
437  )
438 
439  # Variance construction
440  doVariance = pexConfig.Field(
441  dtype=bool,
442  doc="Calculate variance?",
443  default=True
444  )
445  gain = pexConfig.Field(
446  dtype=float,
447  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
448  default=float("NaN"),
449  )
450  readNoise = pexConfig.Field(
451  dtype=float,
452  doc="The read noise to use if no Detector is present in the Exposure",
453  default=0.0,
454  )
455  doEmpiricalReadNoise = pexConfig.Field(
456  dtype=bool,
457  default=False,
458  doc="Calculate empirical read noise instead of value from AmpInfo data?"
459  )
460 
461  # Linearization.
462  doLinearize = pexConfig.Field(
463  dtype=bool,
464  doc="Correct for nonlinearity of the detector's response?",
465  default=True,
466  )
467 
468  # Crosstalk.
469  doCrosstalk = pexConfig.Field(
470  dtype=bool,
471  doc="Apply intra-CCD crosstalk correction?",
472  default=False,
473  )
474  doCrosstalkBeforeAssemble = pexConfig.Field(
475  dtype=bool,
476  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
477  default=False,
478  )
479  crosstalk = pexConfig.ConfigurableField(
480  target=CrosstalkTask,
481  doc="Intra-CCD crosstalk correction",
482  )
483 
484  # Masking options.
485  doDefect = pexConfig.Field(
486  dtype=bool,
487  doc="Apply correction for CCD defects, e.g. hot pixels?",
488  default=True,
489  )
490  doNanMasking = pexConfig.Field(
491  dtype=bool,
492  doc="Mask NAN pixels?",
493  default=True,
494  )
495  doWidenSaturationTrails = pexConfig.Field(
496  dtype=bool,
497  doc="Widen bleed trails based on their width?",
498  default=True
499  )
500 
501  # Brighter-Fatter correction.
502  doBrighterFatter = pexConfig.Field(
503  dtype=bool,
504  default=False,
505  doc="Apply the brighter fatter correction"
506  )
507  brighterFatterLevel = pexConfig.ChoiceField(
508  dtype=str,
509  default="DETECTOR",
510  doc="The level at which to correct for brighter-fatter.",
511  allowed={
512  "AMP": "Every amplifier treated separately.",
513  "DETECTOR": "One kernel per detector",
514  }
515  )
516  brighterFatterMaxIter = pexConfig.Field(
517  dtype=int,
518  default=10,
519  doc="Maximum number of iterations for the brighter fatter correction"
520  )
521  brighterFatterThreshold = pexConfig.Field(
522  dtype=float,
523  default=1000,
524  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
525  " absolute value of the difference between the current corrected image and the one"
526  " from the previous iteration summed over all the pixels."
527  )
528  brighterFatterApplyGain = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Should the gain be applied when applying the brighter fatter correction?"
532  )
533  brighterFatterMaskGrowSize = pexConfig.Field(
534  dtype=int,
535  default=0,
536  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
537  " when brighter-fatter correction is applied."
538  )
539 
540  # Dark subtraction.
541  doDark = pexConfig.Field(
542  dtype=bool,
543  doc="Apply dark frame correction?",
544  default=True,
545  )
546  darkDataProductName = pexConfig.Field(
547  dtype=str,
548  doc="Name of the dark data product",
549  default="dark",
550  )
551 
552  # Camera-specific stray light removal.
553  doStrayLight = pexConfig.Field(
554  dtype=bool,
555  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
556  default=False,
557  )
558  strayLight = pexConfig.ConfigurableField(
559  target=StrayLightTask,
560  doc="y-band stray light correction"
561  )
562 
563  # Flat correction.
564  doFlat = pexConfig.Field(
565  dtype=bool,
566  doc="Apply flat field correction?",
567  default=True,
568  )
569  flatDataProductName = pexConfig.Field(
570  dtype=str,
571  doc="Name of the flat data product",
572  default="flat",
573  )
574  flatScalingType = pexConfig.ChoiceField(
575  dtype=str,
576  doc="The method for scaling the flat on the fly.",
577  default='USER',
578  allowed={
579  "USER": "Scale by flatUserScale",
580  "MEAN": "Scale by the inverse of the mean",
581  "MEDIAN": "Scale by the inverse of the median",
582  },
583  )
584  flatUserScale = pexConfig.Field(
585  dtype=float,
586  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
587  default=1.0,
588  )
589  doTweakFlat = pexConfig.Field(
590  dtype=bool,
591  doc="Tweak flats to match observed amplifier ratios?",
592  default=False
593  )
594 
595  # Amplifier normalization based on gains instead of using flats configuration.
596  doApplyGains = pexConfig.Field(
597  dtype=bool,
598  doc="Correct the amplifiers for their gains instead of applying flat correction",
599  default=False,
600  )
601  normalizeGains = pexConfig.Field(
602  dtype=bool,
603  doc="Normalize all the amplifiers in each CCD to have the same median value.",
604  default=False,
605  )
606 
607  # Fringe correction.
608  doFringe = pexConfig.Field(
609  dtype=bool,
610  doc="Apply fringe correction?",
611  default=True,
612  )
613  fringe = pexConfig.ConfigurableField(
614  target=FringeTask,
615  doc="Fringe subtraction task",
616  )
617  fringeAfterFlat = pexConfig.Field(
618  dtype=bool,
619  doc="Do fringe subtraction after flat-fielding?",
620  default=True,
621  )
622 
623  # Distortion model application.
624  doAddDistortionModel = pexConfig.Field(
625  dtype=bool,
626  doc="Apply a distortion model based on camera geometry to the WCS?",
627  default=True,
628  deprecated=("Camera geometry is incorporated when reading the raw files."
629  " This option no longer is used, and will be removed after v19.")
630  )
631 
632  # Initial CCD-level background statistics options.
633  doMeasureBackground = pexConfig.Field(
634  dtype=bool,
635  doc="Measure the background level on the reduced image?",
636  default=False,
637  )
638 
639  # Camera-specific masking configuration.
640  doCameraSpecificMasking = pexConfig.Field(
641  dtype=bool,
642  doc="Mask camera-specific bad regions?",
643  default=False,
644  )
645  masking = pexConfig.ConfigurableField(
646  target=MaskingTask,
647  doc="Masking task."
648  )
649 
650  # Interpolation options.
651 
652  doInterpolate = pexConfig.Field(
653  dtype=bool,
654  doc="Interpolate masked pixels?",
655  default=True,
656  )
657  doSaturationInterpolation = pexConfig.Field(
658  dtype=bool,
659  doc="Perform interpolation over pixels masked as saturated?"
660  " NB: This is independent of doSaturation; if that is False this plane"
661  " will likely be blank, resulting in a no-op here.",
662  default=True,
663  )
664  doNanInterpolation = pexConfig.Field(
665  dtype=bool,
666  doc="Perform interpolation over pixels masked as NaN?"
667  " NB: This is independent of doNanMasking; if that is False this plane"
668  " will likely be blank, resulting in a no-op here.",
669  default=True,
670  )
671  doNanInterpAfterFlat = pexConfig.Field(
672  dtype=bool,
673  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
674  "also have to interpolate them before flat-fielding."),
675  default=False,
676  )
677  maskListToInterpolate = pexConfig.ListField(
678  dtype=str,
679  doc="List of mask planes that should be interpolated.",
680  default=['SAT', 'BAD', 'UNMASKEDNAN'],
681  )
682  doSaveInterpPixels = pexConfig.Field(
683  dtype=bool,
684  doc="Save a copy of the pre-interpolated pixel values?",
685  default=False,
686  )
687 
688  # Default photometric calibration options.
689  fluxMag0T1 = pexConfig.DictField(
690  keytype=str,
691  itemtype=float,
692  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
693  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
694  ))
695  )
696  defaultFluxMag0T1 = pexConfig.Field(
697  dtype=float,
698  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
699  default=pow(10.0, 0.4*28.0)
700  )
701 
702  # Vignette correction configuration.
703  doVignette = pexConfig.Field(
704  dtype=bool,
705  doc="Apply vignetting parameters?",
706  default=False,
707  )
708  vignette = pexConfig.ConfigurableField(
709  target=VignetteTask,
710  doc="Vignetting task.",
711  )
712 
713  # Transmission curve configuration.
714  doAttachTransmissionCurve = pexConfig.Field(
715  dtype=bool,
716  default=False,
717  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
718  )
719  doUseOpticsTransmission = pexConfig.Field(
720  dtype=bool,
721  default=True,
722  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
723  )
724  doUseFilterTransmission = pexConfig.Field(
725  dtype=bool,
726  default=True,
727  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
728  )
729  doUseSensorTransmission = pexConfig.Field(
730  dtype=bool,
731  default=True,
732  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
733  )
734  doUseAtmosphereTransmission = pexConfig.Field(
735  dtype=bool,
736  default=True,
737  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
738  )
739 
740  # Illumination correction.
741  doIlluminationCorrection = pexConfig.Field(
742  dtype=bool,
743  default=False,
744  doc="Perform illumination correction?"
745  )
746  illuminationCorrectionDataProductName = pexConfig.Field(
747  dtype=str,
748  doc="Name of the illumination correction data product.",
749  default="illumcor",
750  )
751  illumScale = pexConfig.Field(
752  dtype=float,
753  doc="Scale factor for the illumination correction.",
754  default=1.0,
755  )
756  illumFilters = pexConfig.ListField(
757  dtype=str,
758  default=[],
759  doc="Only perform illumination correction for these filters."
760  )
761 
762  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
763  doWrite = pexConfig.Field(
764  dtype=bool,
765  doc="Persist postISRCCD?",
766  default=True,
767  )
768 
769  def validate(self):
770  super().validate()
771  if self.doFlat and self.doApplyGains:
772  raise ValueError("You may not specify both doFlat and doApplyGains")
773  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
774  self.config.maskListToInterpolate.append("SAT")
775  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
776  self.config.maskListToInterpolate.append("UNMASKEDNAN")
777 
778 
779 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
780  """Apply common instrument signature correction algorithms to a raw frame.
781 
782  The process for correcting imaging data is very similar from
783  camera to camera. This task provides a vanilla implementation of
784  doing these corrections, including the ability to turn certain
785  corrections off if they are not needed. The inputs to the primary
786  method, `run()`, are a raw exposure to be corrected and the
787  calibration data products. The raw input is a single chip sized
788  mosaic of all amps including overscans and other non-science
789  pixels. The method `runDataRef()` identifies and defines the
790  calibration data products, and is intended for use by a
791  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
792  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
793  subclassed for different camera, although the most camera specific
794  methods have been split into subtasks that can be redirected
795  appropriately.
796 
797  The __init__ method sets up the subtasks for ISR processing, using
798  the defaults from `lsst.ip.isr`.
799 
800  Parameters
801  ----------
802  args : `list`
803  Positional arguments passed to the Task constructor. None used at this time.
804  kwargs : `dict`, optional
805  Keyword arguments passed on to the Task constructor. None used at this time.
806  """
807  ConfigClass = IsrTaskConfig
808  _DefaultName = "isr"
809 
810  def __init__(self, **kwargs):
811  super().__init__(**kwargs)
812  self.makeSubtask("assembleCcd")
813  self.makeSubtask("crosstalk")
814  self.makeSubtask("strayLight")
815  self.makeSubtask("fringe")
816  self.makeSubtask("masking")
817  self.makeSubtask("vignette")
818 
819  def runQuantum(self, butlerQC, inputRefs, outputRefs):
820  inputs = butlerQC.get(inputRefs)
821 
822  try:
823  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
824  except Exception as e:
825  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
826  (inputRefs, e))
827 
828  inputs['isGen3'] = True
829 
830  detector = inputs['ccdExposure'].getDetector()
831 
832  if self.doLinearize(detector) is True:
833  if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
834  linearizer = linearize.Linearizer(detector=detector)
835  linearizer.fromYaml(inputs['linearizer'])
836  else:
837  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector)
838  inputs['linearizer'] = linearizer
839 
840  if self.config.doDefect is True:
841  if "defects" in inputs and inputs['defects'] is not None:
842  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
843  # masking expects a list of defects defined by their bounding box
844  if not isinstance(inputs["defects"], Defects):
845  inputs["defects"] = Defects.fromTable(inputs["defects"])
846 
847  # Load the correct style of brighter fatter kernel, and repack
848  # the information as a numpy array.
849  if self.config.doBrighterFatter:
850  brighterFatterKernel = inputs.pop('newBFKernel', None)
851  if brighterFatterKernel is None:
852  brighterFatterKernel = inputs.get('bfKernel', None)
853 
854  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
855  detId = detector.getId()
856  inputs['bfGains'] = brighterFatterKernel.gain
857  # If the kernel is not an ndarray, it's the cp_pipe version
858  # so extract the kernel for this detector, or raise an error
859  if self.config.brighterFatterLevel == 'DETECTOR':
860  if brighterFatterKernel.detectorKernel:
861  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
862  elif brighterFatterKernel.detectorKernelFromAmpKernels:
863  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
864  else:
865  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
866  else:
867  # TODO DM-15631 for implementing this
868  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
869 
870  # Broken: DM-17169
871  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
872  # fixed for non-HSC cameras in the future.
873  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
874  # if self.config.doCrosstalk else None)
875 
876  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
877  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
878  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
879  expId=expId,
880  assembler=self.assembleCcd
881  if self.config.doAssembleIsrExposures else None)
882  else:
883  inputs['fringes'] = pipeBase.Struct(fringes=None)
884 
885  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
886  if 'strayLightData' not in inputs:
887  inputs['strayLightData'] = None
888 
889  outputs = self.run(**inputs)
890  butlerQC.put(outputs, outputRefs)
891 
892  def readIsrData(self, dataRef, rawExposure):
893  """!Retrieve necessary frames for instrument signature removal.
894 
895  Pre-fetching all required ISR data products limits the IO
896  required by the ISR. Any conflict between the calibration data
897  available and that needed for ISR is also detected prior to
898  doing processing, allowing it to fail quickly.
899 
900  Parameters
901  ----------
902  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
903  Butler reference of the detector data to be processed
904  rawExposure : `afw.image.Exposure`
905  The raw exposure that will later be corrected with the
906  retrieved calibration data; should not be modified in this
907  method.
908 
909  Returns
910  -------
911  result : `lsst.pipe.base.Struct`
912  Result struct with components (which may be `None`):
913  - ``bias``: bias calibration frame (`afw.image.Exposure`)
914  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
915  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
916  - ``dark``: dark calibration frame (`afw.image.Exposure`)
917  - ``flat``: flat calibration frame (`afw.image.Exposure`)
918  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
919  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
920  - ``fringes``: `lsst.pipe.base.Struct` with components:
921  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
922  - ``seed``: random seed derived from the ccdExposureId for random
923  number generator (`uint32`).
924  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
925  A ``TransmissionCurve`` that represents the throughput of the optics,
926  to be evaluated in focal-plane coordinates.
927  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
928  A ``TransmissionCurve`` that represents the throughput of the filter
929  itself, to be evaluated in focal-plane coordinates.
930  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
931  A ``TransmissionCurve`` that represents the throughput of the sensor
932  itself, to be evaluated in post-assembly trimmed detector coordinates.
933  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
934  A ``TransmissionCurve`` that represents the throughput of the
935  atmosphere, assumed to be spatially constant.
936  - ``strayLightData`` : `object`
937  An opaque object containing calibration information for
938  stray-light correction. If `None`, no correction will be
939  performed.
940  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
941 
942  Raises
943  ------
944  NotImplementedError :
945  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
946  """
947  try:
948  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
949  dateObs = dateObs.toPython().isoformat()
950  except RuntimeError:
951  self.log.warn("Unable to identify dateObs for rawExposure.")
952  dateObs = None
953 
954  ccd = rawExposure.getDetector()
955  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
956  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
957  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
958  if self.config.doBias else None)
959  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
960  linearizer = (dataRef.get("linearizer", immediate=True)
961  if self.doLinearize(ccd) else None)
962  if isinstance(linearizer, numpy.ndarray):
963  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
964  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
965  if self.config.doCrosstalk else None)
966  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
967  if self.config.doDark else None)
968  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
969  dateObs=dateObs)
970  if self.config.doFlat else None)
971 
972  brighterFatterKernel = None
973  brighterFatterGains = None
974  if self.config.doBrighterFatter is True:
975  try:
976  # Use the new-style cp_pipe version of the kernel if it exists
977  # If using a new-style kernel, always use the self-consistent
978  # gains, i.e. the ones inside the kernel object itself
979  brighterFatterKernel = dataRef.get("brighterFatterKernel")
980  brighterFatterGains = brighterFatterKernel.gain
981  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
982  except NoResults:
983  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
984  brighterFatterKernel = dataRef.get("bfKernel")
985  self.log.info("Old style bright-fatter kernel (np.array) loaded")
986  except NoResults:
987  brighterFatterKernel = None
988  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
989  # If the kernel is not an ndarray, it's the cp_pipe version
990  # so extract the kernel for this detector, or raise an error
991  if self.config.brighterFatterLevel == 'DETECTOR':
992  if brighterFatterKernel.detectorKernel:
993  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
994  elif brighterFatterKernel.detectorKernelFromAmpKernels:
995  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
996  else:
997  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
998  else:
999  # TODO DM-15631 for implementing this
1000  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1001 
1002  defectList = (dataRef.get("defects")
1003  if self.config.doDefect else None)
1004  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1005  if self.config.doAssembleIsrExposures else None)
1006  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1007  else pipeBase.Struct(fringes=None))
1008 
1009  if self.config.doAttachTransmissionCurve:
1010  opticsTransmission = (dataRef.get("transmission_optics")
1011  if self.config.doUseOpticsTransmission else None)
1012  filterTransmission = (dataRef.get("transmission_filter")
1013  if self.config.doUseFilterTransmission else None)
1014  sensorTransmission = (dataRef.get("transmission_sensor")
1015  if self.config.doUseSensorTransmission else None)
1016  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1017  if self.config.doUseAtmosphereTransmission else None)
1018  else:
1019  opticsTransmission = None
1020  filterTransmission = None
1021  sensorTransmission = None
1022  atmosphereTransmission = None
1023 
1024  if self.config.doStrayLight:
1025  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1026  else:
1027  strayLightData = None
1028 
1029  illumMaskedImage = (self.getIsrExposure(dataRef,
1030  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1031  if (self.config.doIlluminationCorrection and
1032  filterName in self.config.illumFilters)
1033  else None)
1034 
1035  # Struct should include only kwargs to run()
1036  return pipeBase.Struct(bias=biasExposure,
1037  linearizer=linearizer,
1038  crosstalkSources=crosstalkSources,
1039  dark=darkExposure,
1040  flat=flatExposure,
1041  bfKernel=brighterFatterKernel,
1042  bfGains=brighterFatterGains,
1043  defects=defectList,
1044  fringes=fringeStruct,
1045  opticsTransmission=opticsTransmission,
1046  filterTransmission=filterTransmission,
1047  sensorTransmission=sensorTransmission,
1048  atmosphereTransmission=atmosphereTransmission,
1049  strayLightData=strayLightData,
1050  illumMaskedImage=illumMaskedImage
1051  )
1052 
1053  @pipeBase.timeMethod
1054  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1055  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1056  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1057  sensorTransmission=None, atmosphereTransmission=None,
1058  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1059  isGen3=False,
1060  ):
1061  """!Perform instrument signature removal on an exposure.
1062 
1063  Steps included in the ISR processing, in order performed, are:
1064  - saturation and suspect pixel masking
1065  - overscan subtraction
1066  - CCD assembly of individual amplifiers
1067  - bias subtraction
1068  - variance image construction
1069  - linearization of non-linear response
1070  - crosstalk masking
1071  - brighter-fatter correction
1072  - dark subtraction
1073  - fringe correction
1074  - stray light subtraction
1075  - flat correction
1076  - masking of known defects and camera specific features
1077  - vignette calculation
1078  - appending transmission curve and distortion model
1079 
1080  Parameters
1081  ----------
1082  ccdExposure : `lsst.afw.image.Exposure`
1083  The raw exposure that is to be run through ISR. The
1084  exposure is modified by this method.
1085  camera : `lsst.afw.cameraGeom.Camera`, optional
1086  The camera geometry for this exposure. Used to select the
1087  distortion model appropriate for this data.
1088  bias : `lsst.afw.image.Exposure`, optional
1089  Bias calibration frame.
1090  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1091  Functor for linearization.
1092  crosstalkSources : `list`, optional
1093  List of possible crosstalk sources.
1094  dark : `lsst.afw.image.Exposure`, optional
1095  Dark calibration frame.
1096  flat : `lsst.afw.image.Exposure`, optional
1097  Flat calibration frame.
1098  bfKernel : `numpy.ndarray`, optional
1099  Brighter-fatter kernel.
1100  bfGains : `dict` of `float`, optional
1101  Gains used to override the detector's nominal gains for the
1102  brighter-fatter correction. A dict keyed by amplifier name for
1103  the detector in question.
1104  defects : `lsst.meas.algorithms.Defects`, optional
1105  List of defects.
1106  fringes : `lsst.pipe.base.Struct`, optional
1107  Struct containing the fringe correction data, with
1108  elements:
1109  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1110  - ``seed``: random seed derived from the ccdExposureId for random
1111  number generator (`uint32`)
1112  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1113  A ``TransmissionCurve`` that represents the throughput of the optics,
1114  to be evaluated in focal-plane coordinates.
1115  filterTransmission : `lsst.afw.image.TransmissionCurve`
1116  A ``TransmissionCurve`` that represents the throughput of the filter
1117  itself, to be evaluated in focal-plane coordinates.
1118  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1119  A ``TransmissionCurve`` that represents the throughput of the sensor
1120  itself, to be evaluated in post-assembly trimmed detector coordinates.
1121  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1122  A ``TransmissionCurve`` that represents the throughput of the
1123  atmosphere, assumed to be spatially constant.
1124  detectorNum : `int`, optional
1125  The integer number for the detector to process.
1126  isGen3 : bool, optional
1127  Flag this call to run() as using the Gen3 butler environment.
1128  strayLightData : `object`, optional
1129  Opaque object containing calibration information for stray-light
1130  correction. If `None`, no correction will be performed.
1131  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1132  Illumination correction image.
1133 
1134  Returns
1135  -------
1136  result : `lsst.pipe.base.Struct`
1137  Result struct with component:
1138  - ``exposure`` : `afw.image.Exposure`
1139  The fully ISR corrected exposure.
1140  - ``outputExposure`` : `afw.image.Exposure`
1141  An alias for `exposure`
1142  - ``ossThumb`` : `numpy.ndarray`
1143  Thumbnail image of the exposure after overscan subtraction.
1144  - ``flattenedThumb`` : `numpy.ndarray`
1145  Thumbnail image of the exposure after flat-field correction.
1146 
1147  Raises
1148  ------
1149  RuntimeError
1150  Raised if a configuration option is set to True, but the
1151  required calibration data has not been specified.
1152 
1153  Notes
1154  -----
1155  The current processed exposure can be viewed by setting the
1156  appropriate lsstDebug entries in the `debug.display`
1157  dictionary. The names of these entries correspond to some of
1158  the IsrTaskConfig Boolean options, with the value denoting the
1159  frame to use. The exposure is shown inside the matching
1160  option check and after the processing of that step has
1161  finished. The steps with debug points are:
1162 
1163  doAssembleCcd
1164  doBias
1165  doCrosstalk
1166  doBrighterFatter
1167  doDark
1168  doFringe
1169  doStrayLight
1170  doFlat
1171 
1172  In addition, setting the "postISRCCD" entry displays the
1173  exposure after all ISR processing has finished.
1174 
1175  """
1176 
1177  if isGen3 is True:
1178  # Gen3 currently cannot automatically do configuration overrides.
1179  # DM-15257 looks to discuss this issue.
1180  # Configure input exposures;
1181  if detectorNum is None:
1182  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1183 
1184  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1185  bias = self.ensureExposure(bias, camera, detectorNum)
1186  dark = self.ensureExposure(dark, camera, detectorNum)
1187  flat = self.ensureExposure(flat, camera, detectorNum)
1188  else:
1189  if isinstance(ccdExposure, ButlerDataRef):
1190  return self.runDataRef(ccdExposure)
1191 
1192  ccd = ccdExposure.getDetector()
1193  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1194 
1195  if not ccd:
1196  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1197  ccd = [FakeAmp(ccdExposure, self.config)]
1198 
1199  # Validate Input
1200  if self.config.doBias and bias is None:
1201  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1202  if self.doLinearize(ccd) and linearizer is None:
1203  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1204  if self.config.doBrighterFatter and bfKernel is None:
1205  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1206  if self.config.doDark and dark is None:
1207  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1208  if self.config.doFlat and flat is None:
1209  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1210  if self.config.doDefect and defects is None:
1211  raise RuntimeError("Must supply defects if config.doDefect=True.")
1212  if (self.config.doFringe and filterName in self.fringe.config.filters and
1213  fringes.fringes is None):
1214  # The `fringes` object needs to be a pipeBase.Struct, as
1215  # we use it as a `dict` for the parameters of
1216  # `FringeTask.run()`. The `fringes.fringes` `list` may
1217  # not be `None` if `doFringe=True`. Otherwise, raise.
1218  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1219  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1220  illumMaskedImage is None):
1221  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1222 
1223  # Begin ISR processing.
1224  if self.config.doConvertIntToFloat:
1225  self.log.info("Converting exposure to floating point values.")
1226  ccdExposure = self.convertIntToFloat(ccdExposure)
1227 
1228  # Amplifier level processing.
1229  overscans = []
1230  for amp in ccd:
1231  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1232  if ccdExposure.getBBox().contains(amp.getBBox()):
1233  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1234  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1235 
1236  if self.config.doOverscan and not badAmp:
1237  # Overscan correction on amp-by-amp basis.
1238  overscanResults = self.overscanCorrection(ccdExposure, amp)
1239  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1240  if overscanResults is not None and \
1241  self.config.qa is not None and self.config.qa.saveStats is True:
1242  if isinstance(overscanResults.overscanFit, float):
1243  qaMedian = overscanResults.overscanFit
1244  qaStdev = float("NaN")
1245  else:
1246  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1247  afwMath.MEDIAN | afwMath.STDEVCLIP)
1248  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1249  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1250 
1251  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1252  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1253  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1254  amp.getName(), qaMedian, qaStdev)
1255  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1256  else:
1257  if badAmp:
1258  self.log.warn("Amplifier %s is bad.", amp.getName())
1259  overscanResults = None
1260 
1261  overscans.append(overscanResults if overscanResults is not None else None)
1262  else:
1263  self.log.info("Skipped OSCAN for %s.", amp.getName())
1264 
1265  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1266  self.log.info("Applying crosstalk correction.")
1267  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1268  self.debugView(ccdExposure, "doCrosstalk")
1269 
1270  if self.config.doAssembleCcd:
1271  self.log.info("Assembling CCD from amplifiers.")
1272  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1273 
1274  if self.config.expectWcs and not ccdExposure.getWcs():
1275  self.log.warn("No WCS found in input exposure.")
1276  self.debugView(ccdExposure, "doAssembleCcd")
1277 
1278  ossThumb = None
1279  if self.config.qa.doThumbnailOss:
1280  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1281 
1282  if self.config.doBias:
1283  self.log.info("Applying bias correction.")
1284  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1285  trimToFit=self.config.doTrimToMatchCalib)
1286  self.debugView(ccdExposure, "doBias")
1287 
1288  if self.config.doVariance:
1289  for amp, overscanResults in zip(ccd, overscans):
1290  if ccdExposure.getBBox().contains(amp.getBBox()):
1291  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1292  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1293  if overscanResults is not None:
1294  self.updateVariance(ampExposure, amp,
1295  overscanImage=overscanResults.overscanImage)
1296  else:
1297  self.updateVariance(ampExposure, amp,
1298  overscanImage=None)
1299  if self.config.qa is not None and self.config.qa.saveStats is True:
1300  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1301  afwMath.MEDIAN | afwMath.STDEVCLIP)
1302  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1303  qaStats.getValue(afwMath.MEDIAN))
1304  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1305  qaStats.getValue(afwMath.STDEVCLIP))
1306  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1307  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1308  qaStats.getValue(afwMath.STDEVCLIP))
1309 
1310  if self.doLinearize(ccd):
1311  self.log.info("Applying linearizer.")
1312  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1313  detector=ccd, log=self.log)
1314 
1315  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1316  self.log.info("Applying crosstalk correction.")
1317  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1318  self.debugView(ccdExposure, "doCrosstalk")
1319 
1320  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1321  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1322  if self.config.doDefect:
1323  self.log.info("Masking defects.")
1324  self.maskDefect(ccdExposure, defects)
1325 
1326  if self.config.numEdgeSuspect > 0:
1327  self.log.info("Masking edges as SUSPECT.")
1328  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1329  maskPlane="SUSPECT")
1330 
1331  if self.config.doNanMasking:
1332  self.log.info("Masking NAN value pixels.")
1333  self.maskNan(ccdExposure)
1334 
1335  if self.config.doWidenSaturationTrails:
1336  self.log.info("Widening saturation trails.")
1337  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1338 
1339  if self.config.doCameraSpecificMasking:
1340  self.log.info("Masking regions for camera specific reasons.")
1341  self.masking.run(ccdExposure)
1342 
1343  if self.config.doBrighterFatter:
1344  # We need to apply flats and darks before we can interpolate, and we
1345  # need to interpolate before we do B-F, but we do B-F without the
1346  # flats and darks applied so we can work in units of electrons or holes.
1347  # This context manager applies and then removes the darks and flats.
1348  #
1349  # We also do not want to interpolate values here, so operate on temporary
1350  # images so we can apply only the BF-correction and roll back the
1351  # interpolation.
1352  interpExp = ccdExposure.clone()
1353  with self.flatContext(interpExp, flat, dark):
1354  isrFunctions.interpolateFromMask(
1355  maskedImage=interpExp.getMaskedImage(),
1356  fwhm=self.config.fwhm,
1357  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1358  maskNameList=self.config.maskListToInterpolate
1359  )
1360  bfExp = interpExp.clone()
1361 
1362  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1363  type(bfKernel), type(bfGains))
1364  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1365  self.config.brighterFatterMaxIter,
1366  self.config.brighterFatterThreshold,
1367  self.config.brighterFatterApplyGain,
1368  bfGains)
1369  if bfResults[1] == self.config.brighterFatterMaxIter:
1370  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1371  bfResults[0])
1372  else:
1373  self.log.info("Finished brighter fatter correction in %d iterations.",
1374  bfResults[1])
1375  image = ccdExposure.getMaskedImage().getImage()
1376  bfCorr = bfExp.getMaskedImage().getImage()
1377  bfCorr -= interpExp.getMaskedImage().getImage()
1378  image += bfCorr
1379 
1380  # Applying the brighter-fatter correction applies a
1381  # convolution to the science image. At the edges this
1382  # convolution may not have sufficient valid pixels to
1383  # produce a valid correction. Mark pixels within the size
1384  # of the brighter-fatter kernel as EDGE to warn of this
1385  # fact.
1386  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1387  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1388  maskPlane="EDGE")
1389 
1390  if self.config.brighterFatterMaskGrowSize > 0:
1391  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1392  for maskPlane in self.config.maskListToInterpolate:
1393  isrFunctions.growMasks(ccdExposure.getMask(),
1394  radius=self.config.brighterFatterMaskGrowSize,
1395  maskNameList=maskPlane,
1396  maskValue=maskPlane)
1397 
1398  self.debugView(ccdExposure, "doBrighterFatter")
1399 
1400  if self.config.doDark:
1401  self.log.info("Applying dark correction.")
1402  self.darkCorrection(ccdExposure, dark)
1403  self.debugView(ccdExposure, "doDark")
1404 
1405  if self.config.doFringe and not self.config.fringeAfterFlat:
1406  self.log.info("Applying fringe correction before flat.")
1407  self.fringe.run(ccdExposure, **fringes.getDict())
1408  self.debugView(ccdExposure, "doFringe")
1409 
1410  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1411  self.log.info("Checking strayLight correction.")
1412  self.strayLight.run(ccdExposure, strayLightData)
1413  self.debugView(ccdExposure, "doStrayLight")
1414 
1415  if self.config.doFlat:
1416  self.log.info("Applying flat correction.")
1417  self.flatCorrection(ccdExposure, flat)
1418  self.debugView(ccdExposure, "doFlat")
1419 
1420  if self.config.doApplyGains:
1421  self.log.info("Applying gain correction instead of flat.")
1422  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1423 
1424  if self.config.doFringe and self.config.fringeAfterFlat:
1425  self.log.info("Applying fringe correction after flat.")
1426  self.fringe.run(ccdExposure, **fringes.getDict())
1427 
1428  if self.config.doVignette:
1429  self.log.info("Constructing Vignette polygon.")
1430  self.vignettePolygon = self.vignette.run(ccdExposure)
1431 
1432  if self.config.vignette.doWriteVignettePolygon:
1433  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1434 
1435  if self.config.doAttachTransmissionCurve:
1436  self.log.info("Adding transmission curves.")
1437  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1438  filterTransmission=filterTransmission,
1439  sensorTransmission=sensorTransmission,
1440  atmosphereTransmission=atmosphereTransmission)
1441 
1442  flattenedThumb = None
1443  if self.config.qa.doThumbnailFlattened:
1444  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1445 
1446  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1447  self.log.info("Performing illumination correction.")
1448  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1449  illumMaskedImage, illumScale=self.config.illumScale,
1450  trimToFit=self.config.doTrimToMatchCalib)
1451 
1452  preInterpExp = None
1453  if self.config.doSaveInterpPixels:
1454  preInterpExp = ccdExposure.clone()
1455 
1456  # Reset and interpolate bad pixels.
1457  #
1458  # Large contiguous bad regions (which should have the BAD mask
1459  # bit set) should have their values set to the image median.
1460  # This group should include defects and bad amplifiers. As the
1461  # area covered by these defects are large, there's little
1462  # reason to expect that interpolation would provide a more
1463  # useful value.
1464  #
1465  # Smaller defects can be safely interpolated after the larger
1466  # regions have had their pixel values reset. This ensures
1467  # that the remaining defects adjacent to bad amplifiers (as an
1468  # example) do not attempt to interpolate extreme values.
1469  if self.config.doSetBadRegions:
1470  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1471  if badPixelCount > 0:
1472  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1473 
1474  if self.config.doInterpolate:
1475  self.log.info("Interpolating masked pixels.")
1476  isrFunctions.interpolateFromMask(
1477  maskedImage=ccdExposure.getMaskedImage(),
1478  fwhm=self.config.fwhm,
1479  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1480  maskNameList=list(self.config.maskListToInterpolate)
1481  )
1482 
1483  self.roughZeroPoint(ccdExposure)
1484 
1485  if self.config.doMeasureBackground:
1486  self.log.info("Measuring background level.")
1487  self.measureBackground(ccdExposure, self.config.qa)
1488 
1489  if self.config.qa is not None and self.config.qa.saveStats is True:
1490  for amp in ccd:
1491  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1492  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1493  afwMath.MEDIAN | afwMath.STDEVCLIP)
1494  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1495  qaStats.getValue(afwMath.MEDIAN))
1496  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1497  qaStats.getValue(afwMath.STDEVCLIP))
1498  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1499  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1500  qaStats.getValue(afwMath.STDEVCLIP))
1501 
1502  self.debugView(ccdExposure, "postISRCCD")
1503 
1504  return pipeBase.Struct(
1505  exposure=ccdExposure,
1506  ossThumb=ossThumb,
1507  flattenedThumb=flattenedThumb,
1508 
1509  preInterpolatedExposure=preInterpExp,
1510  outputExposure=ccdExposure,
1511  outputOssThumbnail=ossThumb,
1512  outputFlattenedThumbnail=flattenedThumb,
1513  )
1514 
1515  @pipeBase.timeMethod
1516  def runDataRef(self, sensorRef):
1517  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1518 
1519  This method contains the `CmdLineTask` interface to the ISR
1520  processing. All IO is handled here, freeing the `run()` method
1521  to manage only pixel-level calculations. The steps performed
1522  are:
1523  - Read in necessary detrending/isr/calibration data.
1524  - Process raw exposure in `run()`.
1525  - Persist the ISR-corrected exposure as "postISRCCD" if
1526  config.doWrite=True.
1527 
1528  Parameters
1529  ----------
1530  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1531  DataRef of the detector data to be processed
1532 
1533  Returns
1534  -------
1535  result : `lsst.pipe.base.Struct`
1536  Result struct with component:
1537  - ``exposure`` : `afw.image.Exposure`
1538  The fully ISR corrected exposure.
1539 
1540  Raises
1541  ------
1542  RuntimeError
1543  Raised if a configuration option is set to True, but the
1544  required calibration data does not exist.
1545 
1546  """
1547  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1548 
1549  ccdExposure = sensorRef.get(self.config.datasetType)
1550 
1551  camera = sensorRef.get("camera")
1552  isrData = self.readIsrData(sensorRef, ccdExposure)
1553 
1554  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1555 
1556  if self.config.doWrite:
1557  sensorRef.put(result.exposure, "postISRCCD")
1558  if result.preInterpolatedExposure is not None:
1559  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1560  if result.ossThumb is not None:
1561  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1562  if result.flattenedThumb is not None:
1563  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1564 
1565  return result
1566 
1567  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1568  """!Retrieve a calibration dataset for removing instrument signature.
1569 
1570  Parameters
1571  ----------
1572 
1573  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1574  DataRef of the detector data to find calibration datasets
1575  for.
1576  datasetType : `str`
1577  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1578  dateObs : `str`, optional
1579  Date of the observation. Used to correct butler failures
1580  when using fallback filters.
1581  immediate : `Bool`
1582  If True, disable butler proxies to enable error handling
1583  within this routine.
1584 
1585  Returns
1586  -------
1587  exposure : `lsst.afw.image.Exposure`
1588  Requested calibration frame.
1589 
1590  Raises
1591  ------
1592  RuntimeError
1593  Raised if no matching calibration frame can be found.
1594  """
1595  try:
1596  exp = dataRef.get(datasetType, immediate=immediate)
1597  except Exception as exc1:
1598  if not self.config.fallbackFilterName:
1599  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1600  try:
1601  if self.config.useFallbackDate and dateObs:
1602  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1603  dateObs=dateObs, immediate=immediate)
1604  else:
1605  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1606  except Exception as exc2:
1607  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1608  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1609  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1610 
1611  if self.config.doAssembleIsrExposures:
1612  exp = self.assembleCcd.assembleCcd(exp)
1613  return exp
1614 
1615  def ensureExposure(self, inputExp, camera, detectorNum):
1616  """Ensure that the data returned by Butler is a fully constructed exposure.
1617 
1618  ISR requires exposure-level image data for historical reasons, so if we did
1619  not recieve that from Butler, construct it from what we have, modifying the
1620  input in place.
1621 
1622  Parameters
1623  ----------
1624  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1625  `lsst.afw.image.ImageF`
1626  The input data structure obtained from Butler.
1627  camera : `lsst.afw.cameraGeom.camera`
1628  The camera associated with the image. Used to find the appropriate
1629  detector.
1630  detectorNum : `int`
1631  The detector this exposure should match.
1632 
1633  Returns
1634  -------
1635  inputExp : `lsst.afw.image.Exposure`
1636  The re-constructed exposure, with appropriate detector parameters.
1637 
1638  Raises
1639  ------
1640  TypeError
1641  Raised if the input data cannot be used to construct an exposure.
1642  """
1643  if isinstance(inputExp, afwImage.DecoratedImageU):
1644  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1645  elif isinstance(inputExp, afwImage.ImageF):
1646  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1647  elif isinstance(inputExp, afwImage.MaskedImageF):
1648  inputExp = afwImage.makeExposure(inputExp)
1649  elif isinstance(inputExp, afwImage.Exposure):
1650  pass
1651  elif inputExp is None:
1652  # Assume this will be caught by the setup if it is a problem.
1653  return inputExp
1654  else:
1655  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1656  (type(inputExp), ))
1657 
1658  if inputExp.getDetector() is None:
1659  inputExp.setDetector(camera[detectorNum])
1660 
1661  return inputExp
1662 
1663  def convertIntToFloat(self, exposure):
1664  """Convert exposure image from uint16 to float.
1665 
1666  If the exposure does not need to be converted, the input is
1667  immediately returned. For exposures that are converted to use
1668  floating point pixels, the variance is set to unity and the
1669  mask to zero.
1670 
1671  Parameters
1672  ----------
1673  exposure : `lsst.afw.image.Exposure`
1674  The raw exposure to be converted.
1675 
1676  Returns
1677  -------
1678  newexposure : `lsst.afw.image.Exposure`
1679  The input ``exposure``, converted to floating point pixels.
1680 
1681  Raises
1682  ------
1683  RuntimeError
1684  Raised if the exposure type cannot be converted to float.
1685 
1686  """
1687  if isinstance(exposure, afwImage.ExposureF):
1688  # Nothing to be done
1689  self.log.debug("Exposure already of type float.")
1690  return exposure
1691  if not hasattr(exposure, "convertF"):
1692  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1693 
1694  newexposure = exposure.convertF()
1695  newexposure.variance[:] = 1
1696  newexposure.mask[:] = 0x0
1697 
1698  return newexposure
1699 
1700  def maskAmplifier(self, ccdExposure, amp, defects):
1701  """Identify bad amplifiers, saturated and suspect pixels.
1702 
1703  Parameters
1704  ----------
1705  ccdExposure : `lsst.afw.image.Exposure`
1706  Input exposure to be masked.
1707  amp : `lsst.afw.table.AmpInfoCatalog`
1708  Catalog of parameters defining the amplifier on this
1709  exposure to mask.
1710  defects : `lsst.meas.algorithms.Defects`
1711  List of defects. Used to determine if the entire
1712  amplifier is bad.
1713 
1714  Returns
1715  -------
1716  badAmp : `Bool`
1717  If this is true, the entire amplifier area is covered by
1718  defects and unusable.
1719 
1720  """
1721  maskedImage = ccdExposure.getMaskedImage()
1722 
1723  badAmp = False
1724 
1725  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1726  # comparison with current defects definition.
1727  if defects is not None:
1728  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1729 
1730  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1731  # association with pixels in current ccdExposure).
1732  if badAmp:
1733  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1734  afwImage.PARENT)
1735  maskView = dataView.getMask()
1736  maskView |= maskView.getPlaneBitMask("BAD")
1737  del maskView
1738  return badAmp
1739 
1740  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1741  # Saturation and suspect pixels can be masked now, though.
1742  limits = dict()
1743  if self.config.doSaturation and not badAmp:
1744  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1745  if self.config.doSuspect and not badAmp:
1746  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1747  if math.isfinite(self.config.saturation):
1748  limits.update({self.config.saturatedMaskName: self.config.saturation})
1749 
1750  for maskName, maskThreshold in limits.items():
1751  if not math.isnan(maskThreshold):
1752  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1753  isrFunctions.makeThresholdMask(
1754  maskedImage=dataView,
1755  threshold=maskThreshold,
1756  growFootprints=0,
1757  maskName=maskName
1758  )
1759 
1760  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1761  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1762  afwImage.PARENT)
1763  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1764  self.config.suspectMaskName])
1765  if numpy.all(maskView.getArray() & maskVal > 0):
1766  badAmp = True
1767  maskView |= maskView.getPlaneBitMask("BAD")
1768 
1769  return badAmp
1770 
1771  def overscanCorrection(self, ccdExposure, amp):
1772  """Apply overscan correction in place.
1773 
1774  This method does initial pixel rejection of the overscan
1775  region. The overscan can also be optionally segmented to
1776  allow for discontinuous overscan responses to be fit
1777  separately. The actual overscan subtraction is performed by
1778  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1779  which is called here after the amplifier is preprocessed.
1780 
1781  Parameters
1782  ----------
1783  ccdExposure : `lsst.afw.image.Exposure`
1784  Exposure to have overscan correction performed.
1785  amp : `lsst.afw.table.AmpInfoCatalog`
1786  The amplifier to consider while correcting the overscan.
1787 
1788  Returns
1789  -------
1790  overscanResults : `lsst.pipe.base.Struct`
1791  Result struct with components:
1792  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1793  Value or fit subtracted from the amplifier image data.
1794  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1795  Value or fit subtracted from the overscan image data.
1796  - ``overscanImage`` : `lsst.afw.image.Image`
1797  Image of the overscan region with the overscan
1798  correction applied. This quantity is used to estimate
1799  the amplifier read noise empirically.
1800 
1801  Raises
1802  ------
1803  RuntimeError
1804  Raised if the ``amp`` does not contain raw pixel information.
1805 
1806  See Also
1807  --------
1808  lsst.ip.isr.isrFunctions.overscanCorrection
1809  """
1810  if not amp.getHasRawInfo():
1811  raise RuntimeError("This method must be executed on an amp with raw information.")
1812 
1813  if amp.getRawHorizontalOverscanBBox().isEmpty():
1814  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1815  return None
1816 
1817  statControl = afwMath.StatisticsControl()
1818  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1819 
1820  # Determine the bounding boxes
1821  dataBBox = amp.getRawDataBBox()
1822  oscanBBox = amp.getRawHorizontalOverscanBBox()
1823  dx0 = 0
1824  dx1 = 0
1825 
1826  prescanBBox = amp.getRawPrescanBBox()
1827  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1828  dx0 += self.config.overscanNumLeadingColumnsToSkip
1829  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1830  else:
1831  dx0 += self.config.overscanNumTrailingColumnsToSkip
1832  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1833 
1834  # Determine if we need to work on subregions of the amplifier and overscan.
1835  imageBBoxes = []
1836  overscanBBoxes = []
1837 
1838  if ((self.config.overscanBiasJump and
1839  self.config.overscanBiasJumpLocation) and
1840  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1841  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1842  self.config.overscanBiasJumpDevices)):
1843  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1844  yLower = self.config.overscanBiasJumpLocation
1845  yUpper = dataBBox.getHeight() - yLower
1846  else:
1847  yUpper = self.config.overscanBiasJumpLocation
1848  yLower = dataBBox.getHeight() - yUpper
1849 
1850  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1851  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1852  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1853  lsst.geom.Extent2I(dx0, 0),
1854  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1855  yLower)))
1856 
1857  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1858  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1859  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1860  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1861  yUpper)))
1862  else:
1863  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1864  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1865  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1866  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1867  oscanBBox.getHeight())))
1868 
1869  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1870  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1871  ampImage = ccdExposure.maskedImage[imageBBox]
1872  overscanImage = ccdExposure.maskedImage[overscanBBox]
1873 
1874  overscanArray = overscanImage.image.array
1875  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1876  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1877  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1878 
1879  statControl = afwMath.StatisticsControl()
1880  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1881 
1882  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1883  overscanImage=overscanImage,
1884  fitType=self.config.overscanFitType,
1885  order=self.config.overscanOrder,
1886  collapseRej=self.config.overscanNumSigmaClip,
1887  statControl=statControl,
1888  overscanIsInt=self.config.overscanIsInt
1889  )
1890 
1891  # Measure average overscan levels and record them in the metadata.
1892  levelStat = afwMath.MEDIAN
1893  sigmaStat = afwMath.STDEVCLIP
1894 
1895  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1896  self.config.qa.flatness.nIter)
1897  metadata = ccdExposure.getMetadata()
1898  ampNum = amp.getName()
1899  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1900  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1901  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1902  else:
1903  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1904  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1905  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1906 
1907  return overscanResults
1908 
1909  def updateVariance(self, ampExposure, amp, overscanImage=None):
1910  """Set the variance plane using the amplifier gain and read noise
1911 
1912  The read noise is calculated from the ``overscanImage`` if the
1913  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1914  the value from the amplifier data is used.
1915 
1916  Parameters
1917  ----------
1918  ampExposure : `lsst.afw.image.Exposure`
1919  Exposure to process.
1920  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1921  Amplifier detector data.
1922  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1923  Image of overscan, required only for empirical read noise.
1924 
1925  See also
1926  --------
1927  lsst.ip.isr.isrFunctions.updateVariance
1928  """
1929  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1930  gain = amp.getGain()
1931 
1932  if math.isnan(gain):
1933  gain = 1.0
1934  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1935  elif gain <= 0:
1936  patchedGain = 1.0
1937  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1938  amp.getName(), gain, patchedGain)
1939  gain = patchedGain
1940 
1941  if self.config.doEmpiricalReadNoise and overscanImage is None:
1942  self.log.info("Overscan is none for EmpiricalReadNoise.")
1943 
1944  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1945  stats = afwMath.StatisticsControl()
1946  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1947  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1948  self.log.info("Calculated empirical read noise for amp %s: %f.",
1949  amp.getName(), readNoise)
1950  else:
1951  readNoise = amp.getReadNoise()
1952 
1953  isrFunctions.updateVariance(
1954  maskedImage=ampExposure.getMaskedImage(),
1955  gain=gain,
1956  readNoise=readNoise,
1957  )
1958 
1959  def darkCorrection(self, exposure, darkExposure, invert=False):
1960  """!Apply dark correction in place.
1961 
1962  Parameters
1963  ----------
1964  exposure : `lsst.afw.image.Exposure`
1965  Exposure to process.
1966  darkExposure : `lsst.afw.image.Exposure`
1967  Dark exposure of the same size as ``exposure``.
1968  invert : `Bool`, optional
1969  If True, re-add the dark to an already corrected image.
1970 
1971  Raises
1972  ------
1973  RuntimeError
1974  Raised if either ``exposure`` or ``darkExposure`` do not
1975  have their dark time defined.
1976 
1977  See Also
1978  --------
1979  lsst.ip.isr.isrFunctions.darkCorrection
1980  """
1981  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1982  if math.isnan(expScale):
1983  raise RuntimeError("Exposure darktime is NAN.")
1984  if darkExposure.getInfo().getVisitInfo() is not None \
1985  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
1986  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1987  else:
1988  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1989  # so getDarkTime() does not exist.
1990  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1991  darkScale = 1.0
1992 
1993  isrFunctions.darkCorrection(
1994  maskedImage=exposure.getMaskedImage(),
1995  darkMaskedImage=darkExposure.getMaskedImage(),
1996  expScale=expScale,
1997  darkScale=darkScale,
1998  invert=invert,
1999  trimToFit=self.config.doTrimToMatchCalib
2000  )
2001 
2002  def doLinearize(self, detector):
2003  """!Check if linearization is needed for the detector cameraGeom.
2004 
2005  Checks config.doLinearize and the linearity type of the first
2006  amplifier.
2007 
2008  Parameters
2009  ----------
2010  detector : `lsst.afw.cameraGeom.Detector`
2011  Detector to get linearity type from.
2012 
2013  Returns
2014  -------
2015  doLinearize : `Bool`
2016  If True, linearization should be performed.
2017  """
2018  return self.config.doLinearize and \
2019  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2020 
2021  def flatCorrection(self, exposure, flatExposure, invert=False):
2022  """!Apply flat correction in place.
2023 
2024  Parameters
2025  ----------
2026  exposure : `lsst.afw.image.Exposure`
2027  Exposure to process.
2028  flatExposure : `lsst.afw.image.Exposure`
2029  Flat exposure of the same size as ``exposure``.
2030  invert : `Bool`, optional
2031  If True, unflatten an already flattened image.
2032 
2033  See Also
2034  --------
2035  lsst.ip.isr.isrFunctions.flatCorrection
2036  """
2037  isrFunctions.flatCorrection(
2038  maskedImage=exposure.getMaskedImage(),
2039  flatMaskedImage=flatExposure.getMaskedImage(),
2040  scalingType=self.config.flatScalingType,
2041  userScale=self.config.flatUserScale,
2042  invert=invert,
2043  trimToFit=self.config.doTrimToMatchCalib
2044  )
2045 
2046  def saturationDetection(self, exposure, amp):
2047  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2048 
2049  Parameters
2050  ----------
2051  exposure : `lsst.afw.image.Exposure`
2052  Exposure to process. Only the amplifier DataSec is processed.
2053  amp : `lsst.afw.table.AmpInfoCatalog`
2054  Amplifier detector data.
2055 
2056  See Also
2057  --------
2058  lsst.ip.isr.isrFunctions.makeThresholdMask
2059  """
2060  if not math.isnan(amp.getSaturation()):
2061  maskedImage = exposure.getMaskedImage()
2062  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2063  isrFunctions.makeThresholdMask(
2064  maskedImage=dataView,
2065  threshold=amp.getSaturation(),
2066  growFootprints=0,
2067  maskName=self.config.saturatedMaskName,
2068  )
2069 
2070  def saturationInterpolation(self, exposure):
2071  """!Interpolate over saturated pixels, in place.
2072 
2073  This method should be called after `saturationDetection`, to
2074  ensure that the saturated pixels have been identified in the
2075  SAT mask. It should also be called after `assembleCcd`, since
2076  saturated regions may cross amplifier boundaries.
2077 
2078  Parameters
2079  ----------
2080  exposure : `lsst.afw.image.Exposure`
2081  Exposure to process.
2082 
2083  See Also
2084  --------
2085  lsst.ip.isr.isrTask.saturationDetection
2086  lsst.ip.isr.isrFunctions.interpolateFromMask
2087  """
2088  isrFunctions.interpolateFromMask(
2089  maskedImage=exposure.getMaskedImage(),
2090  fwhm=self.config.fwhm,
2091  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2092  maskNameList=list(self.config.saturatedMaskName),
2093  )
2094 
2095  def suspectDetection(self, exposure, amp):
2096  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2097 
2098  Parameters
2099  ----------
2100  exposure : `lsst.afw.image.Exposure`
2101  Exposure to process. Only the amplifier DataSec is processed.
2102  amp : `lsst.afw.table.AmpInfoCatalog`
2103  Amplifier detector data.
2104 
2105  See Also
2106  --------
2107  lsst.ip.isr.isrFunctions.makeThresholdMask
2108 
2109  Notes
2110  -----
2111  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2112  This is intended to indicate pixels that may be affected by unknown systematics;
2113  for example if non-linearity corrections above a certain level are unstable
2114  then that would be a useful value for suspectLevel. A value of `nan` indicates
2115  that no such level exists and no pixels are to be masked as suspicious.
2116  """
2117  suspectLevel = amp.getSuspectLevel()
2118  if math.isnan(suspectLevel):
2119  return
2120 
2121  maskedImage = exposure.getMaskedImage()
2122  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2123  isrFunctions.makeThresholdMask(
2124  maskedImage=dataView,
2125  threshold=suspectLevel,
2126  growFootprints=0,
2127  maskName=self.config.suspectMaskName,
2128  )
2129 
2130  def maskDefect(self, exposure, defectBaseList):
2131  """!Mask defects using mask plane "BAD", in place.
2132 
2133  Parameters
2134  ----------
2135  exposure : `lsst.afw.image.Exposure`
2136  Exposure to process.
2137  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2138  `lsst.afw.image.DefectBase`.
2139  List of defects to mask.
2140 
2141  Notes
2142  -----
2143  Call this after CCD assembly, since defects may cross amplifier boundaries.
2144  """
2145  maskedImage = exposure.getMaskedImage()
2146  if not isinstance(defectBaseList, Defects):
2147  # Promotes DefectBase to Defect
2148  defectList = Defects(defectBaseList)
2149  else:
2150  defectList = defectBaseList
2151  defectList.maskPixels(maskedImage, maskName="BAD")
2152 
2153  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2154  """!Mask edge pixels with applicable mask plane.
2155 
2156  Parameters
2157  ----------
2158  exposure : `lsst.afw.image.Exposure`
2159  Exposure to process.
2160  numEdgePixels : `int`, optional
2161  Number of edge pixels to mask.
2162  maskPlane : `str`, optional
2163  Mask plane name to use.
2164  """
2165  maskedImage = exposure.getMaskedImage()
2166  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2167 
2168  if numEdgePixels > 0:
2169  goodBBox = maskedImage.getBBox()
2170  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2171  goodBBox.grow(-numEdgePixels)
2172  # Mask pixels outside goodBBox
2173  SourceDetectionTask.setEdgeBits(
2174  maskedImage,
2175  goodBBox,
2176  maskBitMask
2177  )
2178 
2179  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2180  """Mask and interpolate defects using mask plane "BAD", in place.
2181 
2182  Parameters
2183  ----------
2184  exposure : `lsst.afw.image.Exposure`
2185  Exposure to process.
2186  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2187  `lsst.afw.image.DefectBase`.
2188  List of defects to mask and interpolate.
2189 
2190  See Also
2191  --------
2192  lsst.ip.isr.isrTask.maskDefect()
2193  """
2194  self.maskDefect(exposure, defectBaseList)
2195  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2196  maskPlane="SUSPECT")
2197  isrFunctions.interpolateFromMask(
2198  maskedImage=exposure.getMaskedImage(),
2199  fwhm=self.config.fwhm,
2200  growSaturatedFootprints=0,
2201  maskNameList=["BAD"],
2202  )
2203 
2204  def maskNan(self, exposure):
2205  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2206 
2207  Parameters
2208  ----------
2209  exposure : `lsst.afw.image.Exposure`
2210  Exposure to process.
2211 
2212  Notes
2213  -----
2214  We mask over all NaNs, including those that are masked with
2215  other bits (because those may or may not be interpolated over
2216  later, and we want to remove all NaNs). Despite this
2217  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2218  the historical name.
2219  """
2220  maskedImage = exposure.getMaskedImage()
2221 
2222  # Find and mask NaNs
2223  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2224  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2225  numNans = maskNans(maskedImage, maskVal)
2226  self.metadata.set("NUMNANS", numNans)
2227  if numNans > 0:
2228  self.log.warn("There were %d unmasked NaNs.", numNans)
2229 
2230  def maskAndInterpolateNan(self, exposure):
2231  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2232 
2233  Parameters
2234  ----------
2235  exposure : `lsst.afw.image.Exposure`
2236  Exposure to process.
2237 
2238  See Also
2239  --------
2240  lsst.ip.isr.isrTask.maskNan()
2241  """
2242  self.maskNan(exposure)
2243  isrFunctions.interpolateFromMask(
2244  maskedImage=exposure.getMaskedImage(),
2245  fwhm=self.config.fwhm,
2246  growSaturatedFootprints=0,
2247  maskNameList=["UNMASKEDNAN"],
2248  )
2249 
2250  def measureBackground(self, exposure, IsrQaConfig=None):
2251  """Measure the image background in subgrids, for quality control purposes.
2252 
2253  Parameters
2254  ----------
2255  exposure : `lsst.afw.image.Exposure`
2256  Exposure to process.
2257  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2258  Configuration object containing parameters on which background
2259  statistics and subgrids to use.
2260  """
2261  if IsrQaConfig is not None:
2262  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2263  IsrQaConfig.flatness.nIter)
2264  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2265  statsControl.setAndMask(maskVal)
2266  maskedImage = exposure.getMaskedImage()
2267  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2268  skyLevel = stats.getValue(afwMath.MEDIAN)
2269  skySigma = stats.getValue(afwMath.STDEVCLIP)
2270  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2271  metadata = exposure.getMetadata()
2272  metadata.set('SKYLEVEL', skyLevel)
2273  metadata.set('SKYSIGMA', skySigma)
2274 
2275  # calcluating flatlevel over the subgrids
2276  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2277  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2278  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2279  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2280  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2281  skyLevels = numpy.zeros((nX, nY))
2282 
2283  for j in range(nY):
2284  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2285  for i in range(nX):
2286  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2287 
2288  xLLC = xc - meshXHalf
2289  yLLC = yc - meshYHalf
2290  xURC = xc + meshXHalf - 1
2291  yURC = yc + meshYHalf - 1
2292 
2293  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2294  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2295 
2296  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2297 
2298  good = numpy.where(numpy.isfinite(skyLevels))
2299  skyMedian = numpy.median(skyLevels[good])
2300  flatness = (skyLevels[good] - skyMedian) / skyMedian
2301  flatness_rms = numpy.std(flatness)
2302  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2303 
2304  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2305  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2306  nX, nY, flatness_pp, flatness_rms)
2307 
2308  metadata.set('FLATNESS_PP', float(flatness_pp))
2309  metadata.set('FLATNESS_RMS', float(flatness_rms))
2310  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2311  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2312  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2313 
2314  def roughZeroPoint(self, exposure):
2315  """Set an approximate magnitude zero point for the exposure.
2316 
2317  Parameters
2318  ----------
2319  exposure : `lsst.afw.image.Exposure`
2320  Exposure to process.
2321  """
2322  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2323  if filterName in self.config.fluxMag0T1:
2324  fluxMag0 = self.config.fluxMag0T1[filterName]
2325  else:
2326  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2327  fluxMag0 = self.config.defaultFluxMag0T1
2328 
2329  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2330  if not expTime > 0: # handle NaN as well as <= 0
2331  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2332  return
2333 
2334  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2335  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2336 
2337  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2338  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2339 
2340  Parameters
2341  ----------
2342  ccdExposure : `lsst.afw.image.Exposure`
2343  Exposure to process.
2344  fpPolygon : `lsst.afw.geom.Polygon`
2345  Polygon in focal plane coordinates.
2346  """
2347  # Get ccd corners in focal plane coordinates
2348  ccd = ccdExposure.getDetector()
2349  fpCorners = ccd.getCorners(FOCAL_PLANE)
2350  ccdPolygon = Polygon(fpCorners)
2351 
2352  # Get intersection of ccd corners with fpPolygon
2353  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2354 
2355  # Transform back to pixel positions and build new polygon
2356  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2357  validPolygon = Polygon(ccdPoints)
2358  ccdExposure.getInfo().setValidPolygon(validPolygon)
2359 
2360  @contextmanager
2361  def flatContext(self, exp, flat, dark=None):
2362  """Context manager that applies and removes flats and darks,
2363  if the task is configured to apply them.
2364 
2365  Parameters
2366  ----------
2367  exp : `lsst.afw.image.Exposure`
2368  Exposure to process.
2369  flat : `lsst.afw.image.Exposure`
2370  Flat exposure the same size as ``exp``.
2371  dark : `lsst.afw.image.Exposure`, optional
2372  Dark exposure the same size as ``exp``.
2373 
2374  Yields
2375  ------
2376  exp : `lsst.afw.image.Exposure`
2377  The flat and dark corrected exposure.
2378  """
2379  if self.config.doDark and dark is not None:
2380  self.darkCorrection(exp, dark)
2381  if self.config.doFlat:
2382  self.flatCorrection(exp, flat)
2383  try:
2384  yield exp
2385  finally:
2386  if self.config.doFlat:
2387  self.flatCorrection(exp, flat, invert=True)
2388  if self.config.doDark and dark is not None:
2389  self.darkCorrection(exp, dark, invert=True)
2390 
2391  def debugView(self, exposure, stepname):
2392  """Utility function to examine ISR exposure at different stages.
2393 
2394  Parameters
2395  ----------
2396  exposure : `lsst.afw.image.Exposure`
2397  Exposure to view.
2398  stepname : `str`
2399  State of processing to view.
2400  """
2401  frame = getDebugFrame(self._display, stepname)
2402  if frame:
2403  display = getDisplay(frame)
2404  display.scale('asinh', 'zscale')
2405  display.mtv(exposure)
2406  prompt = "Press Enter to continue [c]... "
2407  while True:
2408  ans = input(prompt).lower()
2409  if ans in ("", "c",):
2410  break
2411 
2412 
2413 class FakeAmp(object):
2414  """A Detector-like object that supports returning gain and saturation level
2415 
2416  This is used when the input exposure does not have a detector.
2417 
2418  Parameters
2419  ----------
2420  exposure : `lsst.afw.image.Exposure`
2421  Exposure to generate a fake amplifier for.
2422  config : `lsst.ip.isr.isrTaskConfig`
2423  Configuration to apply to the fake amplifier.
2424  """
2425 
2426  def __init__(self, exposure, config):
2427  self._bbox = exposure.getBBox(afwImage.LOCAL)
2429  self._gain = config.gain
2430  self._readNoise = config.readNoise
2431  self._saturation = config.saturation
2432 
2433  def getBBox(self):
2434  return self._bbox
2435 
2436  def getRawBBox(self):
2437  return self._bbox
2438 
2439  def getHasRawInfo(self):
2440  return True # but see getRawHorizontalOverscanBBox()
2441 
2443  return self._RawHorizontalOverscanBBox
2444 
2445  def getGain(self):
2446  return self._gain
2447 
2448  def getReadNoise(self):
2449  return self._readNoise
2450 
2451  def getSaturation(self):
2452  return self._saturation
2453 
2454  def getSuspectLevel(self):
2455  return float("NaN")
2456 
2457 
2458 class RunIsrConfig(pexConfig.Config):
2459  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2460 
2461 
2462 class RunIsrTask(pipeBase.CmdLineTask):
2463  """Task to wrap the default IsrTask to allow it to be retargeted.
2464 
2465  The standard IsrTask can be called directly from a command line
2466  program, but doing so removes the ability of the task to be
2467  retargeted. As most cameras override some set of the IsrTask
2468  methods, this would remove those data-specific methods in the
2469  output post-ISR images. This wrapping class fixes the issue,
2470  allowing identical post-ISR images to be generated by both the
2471  processCcd and isrTask code.
2472  """
2473  ConfigClass = RunIsrConfig
2474  _DefaultName = "runIsr"
2475 
2476  def __init__(self, *args, **kwargs):
2477  super().__init__(*args, **kwargs)
2478  self.makeSubtask("isr")
2479 
2480  def runDataRef(self, dataRef):
2481  """
2482  Parameters
2483  ----------
2484  dataRef : `lsst.daf.persistence.ButlerDataRef`
2485  data reference of the detector data to be processed
2486 
2487  Returns
2488  -------
2489  result : `pipeBase.Struct`
2490  Result struct with component:
2491 
2492  - exposure : `lsst.afw.image.Exposure`
2493  Post-ISR processed exposure.
2494  """
2495  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1516
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2250
def debugView(self, exposure, stepname)
Definition: isrTask.py:2391
def __init__(self, kwargs)
Definition: isrTask.py:810
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1615
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:892
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2153
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:819
def runDataRef(self, dataRef)
Definition: isrTask.py:2480
def __init__(self, args, kwargs)
Definition: isrTask.py:2476
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2314
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2179
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2442
def maskNan(self, exposure)
Definition: isrTask.py:2204
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2130
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1771
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1663
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:2021
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1060
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1959
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:2002
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2337
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1700
def __init__(self, config=None)
Definition: isrTask.py:179
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2361
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1567
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1909
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2230
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2095
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2070
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:2046
def __init__(self, exposure, config)
Definition: isrTask.py:2426