lsst.ip.isr  16.0-15-g1417920
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 
42 from . import isrFunctions
43 from . import isrQa
44 
45 from .assembleCcdTask import AssembleCcdTask
46 from .crosstalk import CrosstalkTask
47 from .fringe import FringeTask
48 from .isr import maskNans
49 from .masking import MaskingTask
50 from .straylight import StrayLightTask
51 from .vignette import VignetteTask
52 
53 __all__ = ["IsrTask", "RunIsrTask"]
54 
55 
56 class IsrTaskConfig(pexConfig.Config):
57  """Configuration parameters for IsrTask.
58 
59  Items are grouped in the order in which they are executed by the task.
60  """
61  # General ISR configuration
62  datasetType = pexConfig.Field(
63  dtype=str,
64  doc="Dataset type for input data; users will typically leave this alone, "
65  "but camera-specific ISR tasks will override it",
66  default="raw",
67  )
68  fallbackFilterName = pexConfig.Field(
69  dtype=str,
70  doc="Fallback default filter name for calibrations.",
71  optional=True)
72  expectWcs = pexConfig.Field(
73  dtype=bool,
74  default=True,
75  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
76  )
77  fwhm = pexConfig.Field(
78  dtype=float,
79  doc="FWHM of PSF in arcseconds.",
80  default=1.0,
81  )
82  qa = pexConfig.ConfigField(
83  dtype=isrQa.IsrQaConfig,
84  doc="QA related configuration options.",
85  )
86 
87  # Image conversion configuration
88  doConvertIntToFloat = pexConfig.Field(
89  dtype=bool,
90  doc="Convert integer raw images to floating point values?",
91  default=True,
92  )
93 
94  # Saturated pixel handling.
95  doSaturation = pexConfig.Field(
96  dtype=bool,
97  doc="Mask saturated pixels?",
98  default=True,
99  )
100  saturatedMaskName = pexConfig.Field(
101  dtype=str,
102  doc="Name of mask plane to use in saturation detection and interpolation",
103  default="SAT",
104  )
105  saturation = pexConfig.Field(
106  dtype=float,
107  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
108  default=float("NaN"),
109  )
110  growSaturationFootprintSize = pexConfig.Field(
111  dtype=int,
112  doc="Number of pixels by which to grow the saturation footprints",
113  default=1,
114  )
115 
116  # Suspect pixel handling.
117  doSuspect = pexConfig.Field(
118  dtype=bool,
119  doc="Mask suspect pixels?",
120  default=True,
121  )
122  suspectMaskName = pexConfig.Field(
123  dtype=str,
124  doc="Name of mask plane to use for suspect pixels",
125  default="SUSPECT",
126  )
127  numEdgeSuspect = pexConfig.Field(
128  dtype=int,
129  doc="Number of edge pixels to be flagged as untrustworthy.",
130  default=0,
131  )
132 
133  # Initial masking options.
134  doSetBadRegions = pexConfig.Field(
135  dtype=bool,
136  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
137  default=True,
138  )
139  badStatistic = pexConfig.ChoiceField(
140  dtype=str,
141  doc="How to estimate the average value for BAD regions.",
142  default='MEANCLIP',
143  allowed={
144  "MEANCLIP": "Correct using the (clipped) mean of good data",
145  "MEDIAN": "Correct using the median of the good data",
146  },
147  )
148 
149  # Overscan subtraction configuration.
150  doOverscan = pexConfig.Field(
151  dtype=bool,
152  doc="Do overscan subtraction?",
153  default=True,
154  )
155  overscanFitType = pexConfig.ChoiceField(
156  dtype=str,
157  doc="The method for fitting the overscan bias level.",
158  default='MEDIAN',
159  allowed={
160  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
161  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
162  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
163  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
164  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
165  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
166  "MEAN": "Correct using the mean of the overscan region",
167  "MEANCLIP": "Correct using a clipped mean of the overscan region",
168  "MEDIAN": "Correct using the median of the overscan region",
169  },
170  )
171  overscanOrder = pexConfig.Field(
172  dtype=int,
173  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
174  "or number of spline knots if overscan fit type is a spline."),
175  default=1,
176  )
177  overscanNumSigmaClip = pexConfig.Field(
178  dtype=float,
179  doc="Rejection threshold (sigma) for collapsing overscan before fit",
180  default=3.0,
181  )
182  overscanIsInt = pexConfig.Field(
183  dtype=bool,
184  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
185  default=True,
186  )
187  overscanNumLeadingColumnsToSkip = pexConfig.Field(
188  dtype=int,
189  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
190  default=0,
191  )
192  overscanNumTrailingColumnsToSkip = pexConfig.Field(
193  dtype=int,
194  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
195  default=0,
196  )
197  overscanMaxDev = pexConfig.Field(
198  dtype=float,
199  doc="Maximum deviation from the median for overscan",
200  default=1000.0, check=lambda x: x > 0
201  )
202  overscanBiasJump = pexConfig.Field(
203  dtype=bool,
204  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
205  default=False,
206  )
207  overscanBiasJumpKeyword = pexConfig.Field(
208  dtype=str,
209  doc="Header keyword containing information about devices.",
210  default="NO_SUCH_KEY",
211  )
212  overscanBiasJumpDevices = pexConfig.ListField(
213  dtype=str,
214  doc="List of devices that need piecewise overscan correction.",
215  default=(),
216  )
217  overscanBiasJumpLocation = pexConfig.Field(
218  dtype=int,
219  doc="Location of bias jump along y-axis.",
220  default=0,
221  )
222 
223  # Amplifier to CCD assembly configuration
224  doAssembleCcd = pexConfig.Field(
225  dtype=bool,
226  default=True,
227  doc="Assemble amp-level exposures into a ccd-level exposure?"
228  )
229  assembleCcd = pexConfig.ConfigurableField(
230  target=AssembleCcdTask,
231  doc="CCD assembly task",
232  )
233 
234  # General calibration configuration.
235  doAssembleIsrExposures = pexConfig.Field(
236  dtype=bool,
237  default=False,
238  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
239  )
240  doTrimToMatchCalib = pexConfig.Field(
241  dtype=bool,
242  default=False,
243  doc="Trim raw data to match calibration bounding boxes?"
244  )
245 
246  # Bias subtraction.
247  doBias = pexConfig.Field(
248  dtype=bool,
249  doc="Apply bias frame correction?",
250  default=True,
251  )
252  biasDataProductName = pexConfig.Field(
253  dtype=str,
254  doc="Name of the bias data product",
255  default="bias",
256  )
257 
258  # Variance construction
259  doVariance = pexConfig.Field(
260  dtype=bool,
261  doc="Calculate variance?",
262  default=True
263  )
264  gain = pexConfig.Field(
265  dtype=float,
266  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
267  default=float("NaN"),
268  )
269  readNoise = pexConfig.Field(
270  dtype=float,
271  doc="The read noise to use if no Detector is present in the Exposure",
272  default=0.0,
273  )
274  doEmpiricalReadNoise = pexConfig.Field(
275  dtype=bool,
276  default=False,
277  doc="Calculate empirical read noise instead of value from AmpInfo data?"
278  )
279 
280  # Linearization.
281  doLinearize = pexConfig.Field(
282  dtype=bool,
283  doc="Correct for nonlinearity of the detector's response?",
284  default=True,
285  )
286 
287  # Crosstalk.
288  doCrosstalk = pexConfig.Field(
289  dtype=bool,
290  doc="Apply intra-CCD crosstalk correction?",
291  default=False,
292  )
293  doCrosstalkBeforeAssemble = pexConfig.Field(
294  dtype=bool,
295  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
296  default=True,
297  )
298  crosstalk = pexConfig.ConfigurableField(
299  target=CrosstalkTask,
300  doc="Intra-CCD crosstalk correction",
301  )
302 
303  # Masking option prior to brighter-fatter.
304  doWidenSaturationTrails = pexConfig.Field(
305  dtype=bool,
306  doc="Widen bleed trails based on their width?",
307  default=True
308  )
309 
310  # Brighter-Fatter correction.
311  doBrighterFatter = pexConfig.Field(
312  dtype=bool,
313  default=False,
314  doc="Apply the brighter fatter correction"
315  )
316  brighterFatterLevel = pexConfig.ChoiceField(
317  dtype=str,
318  default="DETECTOR",
319  doc="The level at which to correct for brighter-fatter.",
320  allowed={
321  "AMP": "Every amplifier treated separately.",
322  "DETECTOR": "One kernel per detector",
323  }
324  )
325  brighterFatterKernelFile = pexConfig.Field(
326  dtype=str,
327  default='',
328  doc="Kernel file used for the brighter fatter correction"
329  )
330  brighterFatterMaxIter = pexConfig.Field(
331  dtype=int,
332  default=10,
333  doc="Maximum number of iterations for the brighter fatter correction"
334  )
335  brighterFatterThreshold = pexConfig.Field(
336  dtype=float,
337  default=1000,
338  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
339  " absolute value of the difference between the current corrected image and the one"
340  " from the previous iteration summed over all the pixels."
341  )
342  brighterFatterApplyGain = pexConfig.Field(
343  dtype=bool,
344  default=True,
345  doc="Should the gain be applied when applying the brighter fatter correction?"
346  )
347 
348  # Defect and bad pixel correction options.
349  doDefect = pexConfig.Field(
350  dtype=bool,
351  doc="Apply correction for CCD defects, e.g. hot pixels?",
352  default=True,
353  )
354  doSaturationInterpolation = pexConfig.Field(
355  dtype=bool,
356  doc="Perform interpolation over pixels masked as saturated?",
357  default=True,
358  )
359  numEdgeSuspect = pexConfig.Field(
360  dtype=int,
361  doc="Number of edge pixels to be flagged as untrustworthy.",
362  default=0,
363  )
364 
365  # Dark subtraction.
366  doDark = pexConfig.Field(
367  dtype=bool,
368  doc="Apply dark frame correction?",
369  default=True,
370  )
371  darkDataProductName = pexConfig.Field(
372  dtype=str,
373  doc="Name of the dark data product",
374  default="dark",
375  )
376 
377  # Camera-specific stray light removal.
378  doStrayLight = pexConfig.Field(
379  dtype=bool,
380  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
381  default=False,
382  )
383  strayLight = pexConfig.ConfigurableField(
384  target=StrayLightTask,
385  doc="y-band stray light correction"
386  )
387 
388  # Flat correction.
389  doFlat = pexConfig.Field(
390  dtype=bool,
391  doc="Apply flat field correction?",
392  default=True,
393  )
394  flatDataProductName = pexConfig.Field(
395  dtype=str,
396  doc="Name of the flat data product",
397  default="flat",
398  )
399  flatScalingType = pexConfig.ChoiceField(
400  dtype=str,
401  doc="The method for scaling the flat on the fly.",
402  default='USER',
403  allowed={
404  "USER": "Scale by flatUserScale",
405  "MEAN": "Scale by the inverse of the mean",
406  "MEDIAN": "Scale by the inverse of the median",
407  },
408  )
409  flatUserScale = pexConfig.Field(
410  dtype=float,
411  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
412  default=1.0,
413  )
414  doTweakFlat = pexConfig.Field(
415  dtype=bool,
416  doc="Tweak flats to match observed amplifier ratios?",
417  default=False
418  )
419 
420  # Amplifier normalization based on gains instead of using flats configuration.
421  doApplyGains = pexConfig.Field(
422  dtype=bool,
423  doc="Correct the amplifiers for their gains instead of applying flat correction",
424  default=False,
425  )
426  normalizeGains = pexConfig.Field(
427  dtype=bool,
428  doc="Normalize all the amplifiers in each CCD to have the same median value.",
429  default=False,
430  )
431 
432  # Fringe correction.
433  doFringe = pexConfig.Field(
434  dtype=bool,
435  doc="Apply fringe correction?",
436  default=True,
437  )
438  fringe = pexConfig.ConfigurableField(
439  target=FringeTask,
440  doc="Fringe subtraction task",
441  )
442  fringeAfterFlat = pexConfig.Field(
443  dtype=bool,
444  doc="Do fringe subtraction after flat-fielding?",
445  default=True,
446  )
447 
448  # NAN pixel interpolation option.
449  doNanInterpAfterFlat = pexConfig.Field(
450  dtype=bool,
451  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
452  "also have to interpolate them before flat-fielding."),
453  default=False,
454  )
455 
456  # Distortion model application.
457  doAddDistortionModel = pexConfig.Field(
458  dtype=bool,
459  doc="Apply a distortion model based on camera geometry to the WCS?",
460  default=True,
461  )
462 
463  # Initial CCD-level background statistics options.
464  doMeasureBackground = pexConfig.Field(
465  dtype=bool,
466  doc="Measure the background level on the reduced image?",
467  default=False,
468  )
469 
470  # Camera-specific masking configuration.
471  doCameraSpecificMasking = pexConfig.Field(
472  dtype=bool,
473  doc="Mask camera-specific bad regions?",
474  default=False,
475  )
476  masking = pexConfig.ConfigurableField(
477  target=MaskingTask,
478  doc="Masking task."
479  )
480 
481  # Default photometric calibration options.
482  fluxMag0T1 = pexConfig.DictField(
483  keytype=str,
484  itemtype=float,
485  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
486  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
487  ))
488  )
489  defaultFluxMag0T1 = pexConfig.Field(
490  dtype=float,
491  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
492  default=pow(10.0, 0.4*28.0)
493  )
494 
495  # Vignette correction configuration.
496  doVignette = pexConfig.Field(
497  dtype=bool,
498  doc="Apply vignetting parameters?",
499  default=False,
500  )
501  vignette = pexConfig.ConfigurableField(
502  target=VignetteTask,
503  doc="Vignetting task.",
504  )
505 
506  # Transmission curve configuration.
507  doAttachTransmissionCurve = pexConfig.Field(
508  dtype=bool,
509  default=False,
510  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
511  )
512  doUseOpticsTransmission = pexConfig.Field(
513  dtype=bool,
514  default=True,
515  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
516  )
517  doUseFilterTransmission = pexConfig.Field(
518  dtype=bool,
519  default=True,
520  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
521  )
522  doUseSensorTransmission = pexConfig.Field(
523  dtype=bool,
524  default=True,
525  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
526  )
527  doUseAtmosphereTransmission = pexConfig.Field(
528  dtype=bool,
529  default=True,
530  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
531  )
532 
533  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
534  doWrite = pexConfig.Field(
535  dtype=bool,
536  doc="Persist postISRCCD?",
537  default=True,
538  )
539 
540  def validate(self):
541  super().validate()
542  if self.doFlat and self.doApplyGains:
543  raise ValueError("You may not specify both doFlat and doApplyGains")
544 
545 
546 class IsrTask(pipeBase.CmdLineTask):
547  r"""Apply common instrument signature correction algorithms to a raw frame.
548 
549  The process for correcting imaging data is very similar from
550  camera to camera. This task provides a vanilla implementation of
551  doing these corrections, including the ability to turn certain
552  corrections off if they are not needed. The inputs to the primary
553  method, `run()`, are a raw exposure to be corrected and the
554  calibration data products. The raw input is a single chip sized
555  mosaic of all amps including overscans and other non-science
556  pixels. The method `runDataRef()` identifies and defines the
557  calibration data products, and is intended for use by a
558  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
559  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
560  subclassed for different camera, although the most camera specific
561  methods have been split into subtasks that can be redirected
562  appropriately.
563 
564  The __init__ method sets up the subtasks for ISR processing, using
565  the defaults from `lsst.ip.isr`.
566 
567  Parameters
568  ----------
569  args : `list`
570  Positional arguments passed to the Task constructor. None used at this time.
571  kwargs : `dict`, optional
572  Keyword arguments passed on to the Task constructor. None used at this time.
573  """
574  ConfigClass = IsrTaskConfig
575  _DefaultName = "isr"
576 
577  def __init__(self, *args, **kwargs):
578  pipeBase.Task.__init__(self, *args, **kwargs)
579 
580  self.makeSubtask("assembleCcd")
581  self.makeSubtask("crosstalk")
582  self.makeSubtask("strayLight")
583  self.makeSubtask("fringe")
584  self.makeSubtask("masking")
585  self.makeSubtask("vignette")
586 
587  def readIsrData(self, dataRef, rawExposure):
588  """!Retrieve necessary frames for instrument signature removal.
589 
590  Pre-fetching all required ISR data products limits the IO
591  required by the ISR. Any conflict between the calibration data
592  available and that needed for ISR is also detected prior to
593  doing processing, allowing it to fail quickly.
594 
595  Parameters
596  ----------
597  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
598  Butler reference of the detector data to be processed
599  rawExposure : `afw.image.Exposure`
600  The raw exposure that will later be corrected with the
601  retrieved calibration data; should not be modified in this
602  method.
603 
604  Returns
605  -------
606  result : `lsst.pipe.base.Struct`
607  Result struct with components (which may be `None`):
608  - ``bias``: bias calibration frame (`afw.image.Exposure`)
609  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
610  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
611  - ``dark``: dark calibration frame (`afw.image.Exposure`)
612  - ``flat``: flat calibration frame (`afw.image.Exposure`)
613  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
614  - ``defects``: list of defects (`list`)
615  - ``fringes``: `lsst.pipe.base.Struct` with components:
616  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
617  - ``seed``: random seed derived from the ccdExposureId for random
618  number generator (`uint32`)
619  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
620  A ``TransmissionCurve`` that represents the throughput of the optics,
621  to be evaluated in focal-plane coordinates.
622  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
623  A ``TransmissionCurve`` that represents the throughput of the filter
624  itself, to be evaluated in focal-plane coordinates.
625  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
626  A ``TransmissionCurve`` that represents the throughput of the sensor
627  itself, to be evaluated in post-assembly trimmed detector coordinates.
628  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
629  A ``TransmissionCurve`` that represents the throughput of the
630  atmosphere, assumed to be spatially constant.
631 
632  """
633  ccd = rawExposure.getDetector()
634  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
635  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
636  if self.config.doBias else None)
637  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
638  linearizer = (dataRef.get("linearizer", immediate=True)
639  if self.doLinearize(ccd) else None)
640  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
641  if self.config.doCrosstalk else None)
642  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
643  if self.config.doDark else None)
644  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
645  if self.config.doFlat else None)
646  brighterFatterKernel = (dataRef.get("bfKernel")
647  if self.config.doBrighterFatter else None)
648  defectList = (dataRef.get("defects")
649  if self.config.doDefect else None)
650  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
651  if self.config.doAssembleIsrExposures else None)
652  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
653  else pipeBase.Struct(fringes=None))
654 
655  if self.config.doAttachTransmissionCurve:
656  opticsTransmission = (dataRef.get("transmission_optics")
657  if self.config.doUseOpticsTransmission else None)
658  filterTransmission = (dataRef.get("transmission_filter")
659  if self.config.doUseFilterTransmission else None)
660  sensorTransmission = (dataRef.get("transmission_sensor")
661  if self.config.doUseSensorTransmission else None)
662  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
663  if self.config.doUseAtmosphereTransmission else None)
664  else:
665  opticsTransmission = None
666  filterTransmission = None
667  sensorTransmission = None
668  atmosphereTransmission = None
669 
670  # Struct should include only kwargs to run()
671  return pipeBase.Struct(bias=biasExposure,
672  linearizer=linearizer,
673  crosstalkSources=crosstalkSources,
674  dark=darkExposure,
675  flat=flatExposure,
676  bfKernel=brighterFatterKernel,
677  defects=defectList,
678  fringes=fringeStruct,
679  opticsTransmission=opticsTransmission,
680  filterTransmission=filterTransmission,
681  sensorTransmission=sensorTransmission,
682  atmosphereTransmission=atmosphereTransmission,
683  )
684 
685  @pipeBase.timeMethod
686  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
687  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
688  opticsTransmission=None, filterTransmission=None,
689  sensorTransmission=None, atmosphereTransmission=None,
690  ):
691  """!Perform instrument signature removal on an exposure.
692 
693  Steps included in the ISR processing, in order performed, are:
694  - saturation and suspect pixel masking
695  - overscan subtraction
696  - CCD assembly of individual amplifiers
697  - bias subtraction
698  - variance image construction
699  - linearization of non-linear response
700  - crosstalk masking
701  - brighter-fatter correction
702  - dark subtraction
703  - fringe correction
704  - stray light subtraction
705  - flat correction
706  - masking of known defects and camera specific features
707  - vignette calculation
708  - appending transmission curve and distortion model
709 
710  Parameters
711  ----------
712  ccdExposure : `lsst.afw.image.Exposure`
713  The raw exposure that is to be run through ISR. The
714  exposure is modified by this method.
715  camera : `lsst.afw.cameraGeom.Camera`, optional
716  The camera geometry for this exposure. Used to select the
717  distortion model appropriate for this data.
718  bias : `lsst.afw.image.Exposure`, optional
719  Bias calibration frame.
720  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
721  Functor for linearization.
722  crosstalkSources : `list`, optional
723  List of possible crosstalk sources.
724  dark : `lsst.afw.image.Exposure`, optional
725  Dark calibration frame.
726  flat : `lsst.afw.image.Exposure`, optional
727  Flat calibration frame.
728  bfKernel : `numpy.ndarray`, optional
729  Brighter-fatter kernel.
730  defects : `list`, optional
731  List of defects.
732  fringes : `lsst.pipe.base.Struct`, optional
733  Struct containing the fringe correction data, with
734  elements:
735  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
736  - ``seed``: random seed derived from the ccdExposureId for random
737  number generator (`uint32`)
738  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
739  A ``TransmissionCurve`` that represents the throughput of the optics,
740  to be evaluated in focal-plane coordinates.
741  filterTransmission : `lsst.afw.image.TransmissionCurve`
742  A ``TransmissionCurve`` that represents the throughput of the filter
743  itself, to be evaluated in focal-plane coordinates.
744  sensorTransmission : `lsst.afw.image.TransmissionCurve`
745  A ``TransmissionCurve`` that represents the throughput of the sensor
746  itself, to be evaluated in post-assembly trimmed detector coordinates.
747  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
748  A ``TransmissionCurve`` that represents the throughput of the
749  atmosphere, assumed to be spatially constant.
750 
751  Returns
752  -------
753  result : `lsst.pipe.base.Struct`
754  Result struct with component:
755  - ``exposure`` : `afw.image.Exposure`
756  The fully ISR corrected exposure.
757  - ``ossThumb`` : `numpy.ndarray`
758  Thumbnail image of the exposure after overscan subtraction.
759  - ``flattenedThumb`` : `numpy.ndarray`
760  Thumbnail image of the exposure after flat-field correction.
761 
762  Raises
763  ------
764  RuntimeError
765  Raised if a configuration option is set to True, but the
766  required calibration data has not been specified.
767 
768  Notes
769  -----
770  The current processed exposure can be viewed by setting the
771  appropriate lsstDebug entries in the `debug.display`
772  dictionary. The names of these entries correspond to some of
773  the IsrTaskConfig Boolean options, with the value denoting the
774  frame to use. The exposure is shown inside the matching
775  option check and after the processing of that step has
776  finished. The steps with debug points are:
777 
778  doAssembleCcd
779  doBias
780  doCrosstalk
781  doBrighterFatter
782  doDark
783  doFringe
784  doStrayLight
785  doFlat
786 
787  In addition, setting the "postISRCCD" entry displays the
788  exposure after all ISR processing has finished.
789 
790  """
791  if isinstance(ccdExposure, ButlerDataRef):
792  return self.runDataRef(ccdExposure)
793 
794  self.log.info("Performing ISR on exposure %s" %
795  (ccdExposure.getInfo().getVisitInfo().getExposureId()))
796  ccd = ccdExposure.getDetector()
797 
798  if not ccd:
799  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
800  ccd = [FakeAmp(ccdExposure, self.config)]
801 
802  # Validate Input
803  if self.config.doBias and bias is None:
804  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
805  if self.doLinearize(ccd) and linearizer is None:
806  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
807  if self.config.doBrighterFatter and bfKernel is None:
808  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
809  if self.config.doDark and dark is None:
810  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
811  if fringes is None:
812  fringes = pipeBase.Struct(fringes=None)
813  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
814  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
815  if self.config.doFlat and flat is None:
816  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
817  if self.config.doDefect and defects is None:
818  raise RuntimeError("Must supply defects if config.doDefect=True.")
819  if self.config.doAddDistortionModel and camera is None:
820  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
821 
822  # Begin ISR processing.
823  if self.config.doConvertIntToFloat:
824  self.log.info("Converting exposure to floating point values")
825  ccdExposure = self.convertIntToFloat(ccdExposure)
826 
827  # Amplifier level processing.
828  overscans = []
829  for amp in ccd:
830  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
831  if ccdExposure.getBBox().contains(amp.getBBox()):
832  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
833  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
834 
835  if self.config.doOverscan and not badAmp:
836  # Overscan correction on amp-by-amp basis.
837  overscanResults = self.overscanCorrection(ccdExposure, amp)
838  self.log.info("Corrected overscan for amplifier %s" % (amp.getName()))
839  if self.config.qa is not None and self.config.qa.saveStats is True:
840  if isinstance(overscanResults.overscanFit, float):
841  qaMedian = overscanResults.overscanFit
842  qaStdev = float("NaN")
843  else:
844  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
845  afwMath.MEDIAN | afwMath.STDEVCLIP)
846  qaMedian = qaStats.getValue(afwMath.MEDIAN)
847  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
848 
849  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
850  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
851  self.log.info(" Overscan stats for amplifer %s: %f +/- %f" %
852  (amp.getName(), qaMedian, qaStdev))
853  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
854  else:
855  self.log.warn("Amplifier %s is bad." % (amp.getName()))
856  overscanResults = None
857 
858  overscans.append(overscanResults if overscanResults is not None else None)
859  else:
860  self.log.info("Skipped OSCAN")
861 
862  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
863  self.log.info("Applying crosstalk correction.")
864  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
865  self.debugView(ccdExposure, "doCrosstalk")
866 
867  if self.config.doAssembleCcd:
868  self.log.info("Assembling CCD from amplifiers")
869  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
870 
871  if self.config.expectWcs and not ccdExposure.getWcs():
872  self.log.warn("No WCS found in input exposure")
873  self.debugView(ccdExposure, "doAssembleCcd")
874 
875  ossThumb = None
876  if self.config.qa.doThumbnailOss:
877  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
878 
879  if self.config.doBias:
880  self.log.info("Applying bias correction.")
881  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
882  trimToFit=self.config.doTrimToMatchCalib)
883  self.debugView(ccdExposure, "doBias")
884 
885  if self.config.doVariance:
886  for amp, overscanResults in zip(ccd, overscans):
887  if ccdExposure.getBBox().contains(amp.getBBox()):
888  self.log.info("Constructing variance map for amplifer %s" % (amp.getName()))
889  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
890  if overscanResults is not None:
891  self.updateVariance(ampExposure, amp,
892  overscanImage=overscanResults.overscanImage)
893  else:
894  self.updateVariance(ampExposure, amp,
895  overscanImage=None)
896  if self.config.qa is not None and self.config.qa.saveStats is True:
897  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
898  afwMath.MEDIAN | afwMath.STDEVCLIP)
899  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
900  qaStats.getValue(afwMath.MEDIAN))
901  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
902  qaStats.getValue(afwMath.STDEVCLIP))
903  self.log.info(" Variance stats for amplifer %s: %f +/- %f" %
904  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
905  qaStats.getValue(afwMath.STDEVCLIP)))
906 
907  if self.doLinearize(ccd):
908  self.log.info("Applying linearizer.")
909  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
910 
911  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
912  self.log.info("Applying crosstalk correction.")
913  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
914  self.debugView(ccdExposure, "doCrosstalk")
915 
916  if self.config.doWidenSaturationTrails:
917  self.log.info("Widening saturation trails.")
918  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
919 
920  interpolationDone = False
921  if self.config.doBrighterFatter:
922  # We need to apply flats and darks before we can interpolate, and we
923  # need to interpolate before we do B-F, but we do B-F without the
924  # flats and darks applied so we can work in units of electrons or holes.
925  # This context manager applies and then removes the darks and flats.
926  with self.flatContext(ccdExposure, flat, dark):
927  if self.config.doDefect:
928  self.maskAndInterpDefect(ccdExposure, defects)
929 
930  if self.config.doSaturationInterpolation:
931  self.saturationInterpolation(ccdExposure)
932 
933  self.maskAndInterpNan(ccdExposure)
934  interpolationDone = True
935 
936  if self.config.brighterFatterLevel == 'DETECTOR':
937  kernelElement = bfKernel
938  else:
939  # TODO: DM-15631 for implementing this
940  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
941  self.log.info("Applying brighter fatter correction.")
942  isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
943  self.config.brighterFatterMaxIter,
944  self.config.brighterFatterThreshold,
945  self.config.brighterFatterApplyGain,
946  )
947  self.debugView(ccdExposure, "doBrighterFatter")
948 
949  if self.config.doDark:
950  self.log.info("Applying dark correction.")
951  self.darkCorrection(ccdExposure, dark)
952  self.debugView(ccdExposure, "doDark")
953 
954  if self.config.doFringe and not self.config.fringeAfterFlat:
955  self.log.info("Applying fringe correction before flat.")
956  self.fringe.run(ccdExposure, **fringes.getDict())
957  self.debugView(ccdExposure, "doFringe")
958 
959  if self.config.doStrayLight:
960  self.log.info("Applying stray light correction.")
961  self.strayLight.run(ccdExposure)
962  self.debugView(ccdExposure, "doStrayLight")
963 
964  if self.config.doFlat:
965  self.log.info("Applying flat correction.")
966  self.flatCorrection(ccdExposure, flat)
967  self.debugView(ccdExposure, "doFlat")
968 
969  if self.config.doApplyGains:
970  self.log.info("Applying gain correction instead of flat.")
971  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
972 
973  if self.config.doDefect and not interpolationDone:
974  self.log.info("Masking and interpolating defects.")
975  self.maskAndInterpDefect(ccdExposure, defects)
976 
977  if self.config.doSaturation and not interpolationDone:
978  self.log.info("Interpolating saturated pixels.")
979  self.saturationInterpolation(ccdExposure)
980 
981  if self.config.doNanInterpAfterFlat or not interpolationDone:
982  self.log.info("Masking and interpolating NAN value pixels.")
983  self.maskAndInterpNan(ccdExposure)
984 
985  if self.config.doFringe and self.config.fringeAfterFlat:
986  self.log.info("Applying fringe correction after flat.")
987  self.fringe.run(ccdExposure, **fringes.getDict())
988 
989  if self.config.doSetBadRegions:
990  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
991  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
992 
993  flattenedThumb = None
994  if self.config.qa.doThumbnailFlattened:
995  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
996 
997  if self.config.doCameraSpecificMasking:
998  self.log.info("Masking regions for camera specific reasons.")
999  self.masking.run(ccdExposure)
1000 
1001  self.roughZeroPoint(ccdExposure)
1002 
1003  if self.config.doVignette:
1004  self.log.info("Constructing Vignette polygon.")
1005  self.vignettePolygon = self.vignette.run(ccdExposure)
1006 
1007  if self.config.vignette.doWriteVignettePolygon:
1008  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1009 
1010  if self.config.doAttachTransmissionCurve:
1011  self.log.info("Adding transmission curves.")
1012  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1013  filterTransmission=filterTransmission,
1014  sensorTransmission=sensorTransmission,
1015  atmosphereTransmission=atmosphereTransmission)
1016 
1017  if self.config.doAddDistortionModel:
1018  self.log.info("Adding a distortion model to the WCS.")
1019  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1020 
1021  if self.config.doMeasureBackground:
1022  self.log.info("Measuring background level:")
1023  self.measureBackground(ccdExposure, self.config.qa)
1024 
1025  if self.config.qa is not None and self.config.qa.saveStats is True:
1026  for amp in ccd:
1027  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1028  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1029  afwMath.MEDIAN | afwMath.STDEVCLIP)
1030  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1031  qaStats.getValue(afwMath.MEDIAN))
1032  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1033  qaStats.getValue(afwMath.STDEVCLIP))
1034  self.log.info(" Background stats for amplifer %s: %f +/- %f" %
1035  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1036  qaStats.getValue(afwMath.STDEVCLIP)))
1037 
1038  self.debugView(ccdExposure, "postISRCCD")
1039 
1040  return pipeBase.Struct(
1041  exposure=ccdExposure,
1042  ossThumb=ossThumb,
1043  flattenedThumb=flattenedThumb
1044  )
1045 
1046  @pipeBase.timeMethod
1047  def runDataRef(self, sensorRef):
1048  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1049 
1050  This method contains the `CmdLineTask` interface to the ISR
1051  processing. All IO is handled here, freeing the `run()` method
1052  to manage only pixel-level calculations. The steps performed
1053  are:
1054  - Read in necessary detrending/isr/calibration data.
1055  - Process raw exposure in `run()`.
1056  - Persist the ISR-corrected exposure as "postISRCCD" if
1057  config.doWrite=True.
1058 
1059  Parameters
1060  ----------
1061  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1062  DataRef of the detector data to be processed
1063 
1064  Returns
1065  -------
1066  result : `lsst.pipe.base.Struct`
1067  Result struct with component:
1068  - ``exposure`` : `afw.image.Exposure`
1069  The fully ISR corrected exposure.
1070 
1071  Raises
1072  ------
1073  RuntimeError
1074  Raised if a configuration option is set to True, but the
1075  required calibration data does not exist.
1076 
1077  """
1078  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1079  ccdExposure = sensorRef.get(self.config.datasetType)
1080 
1081  camera = sensorRef.get("camera")
1082  if camera is None and self.config.doAddDistortionModel:
1083  raise RuntimeError("config.doAddDistortionModel is True "
1084  "but could not get a camera from the butler")
1085  isrData = self.readIsrData(sensorRef, ccdExposure)
1086 
1087  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1088 
1089  if self.config.doWrite:
1090  sensorRef.put(result.exposure, "postISRCCD")
1091  if result.ossThumb is not None:
1092  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1093  if result.flattenedThumb is not None:
1094  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1095 
1096  return result
1097 
1098  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1099  """!Retrieve a calibration dataset for removing instrument signature.
1100 
1101  Parameters
1102  ----------
1103 
1104  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1105  DataRef of the detector data to find calibration datasets
1106  for.
1107  datasetType : `str`
1108  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1109  immediate : `Bool`
1110  If True, disable butler proxies to enable error handling
1111  within this routine.
1112 
1113  Returns
1114  -------
1115  exposure : `lsst.afw.image.Exposure`
1116  Requested calibration frame.
1117 
1118  Raises
1119  ------
1120  RuntimeError
1121  Raised if no matching calibration frame can be found.
1122  """
1123  try:
1124  exp = dataRef.get(datasetType, immediate=immediate)
1125  except Exception as exc1:
1126  if not self.config.fallbackFilterName:
1127  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1128  try:
1129  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1130  except Exception as exc2:
1131  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1132  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1133  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1134 
1135  if self.config.doAssembleIsrExposures:
1136  exp = self.assembleCcd.assembleCcd(exp)
1137  return exp
1138 
1139  def convertIntToFloat(self, exposure):
1140  """Convert exposure image from uint16 to float.
1141 
1142  If the exposure does not need to be converted, the input is
1143  immediately returned. For exposures that are converted to use
1144  floating point pixels, the variance is set to unity and the
1145  mask to zero.
1146 
1147  Parameters
1148  ----------
1149  exposure : `lsst.afw.image.Exposure`
1150  The raw exposure to be converted.
1151 
1152  Returns
1153  -------
1154  newexposure : `lsst.afw.image.Exposure`
1155  The input ``exposure``, converted to floating point pixels.
1156 
1157  Raises
1158  ------
1159  RuntimeError
1160  Raised if the exposure type cannot be converted to float.
1161 
1162  """
1163  if isinstance(exposure, afwImage.ExposureF):
1164  # Nothing to be done
1165  return exposure
1166  if not hasattr(exposure, "convertF"):
1167  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1168 
1169  newexposure = exposure.convertF()
1170  newexposure.variance[:] = 1
1171  newexposure.mask[:] = 0x0
1172 
1173  return newexposure
1174 
1175  def maskAmplifier(self, ccdExposure, amp, defects):
1176  """Identify bad amplifiers, saturated and suspect pixels.
1177 
1178  Parameters
1179  ----------
1180  ccdExposure : `lsst.afw.image.Exposure`
1181  Input exposure to be masked.
1182  amp : `lsst.afw.table.AmpInfoCatalog`
1183  Catalog of parameters defining the amplifier on this
1184  exposure to mask.
1185  defects : `list`
1186  List of defects. Used to determine if the entire
1187  amplifier is bad.
1188 
1189  Returns
1190  -------
1191  badAmp : `Bool`
1192  If this is true, the entire amplifier area is covered by
1193  defects and unusable.
1194 
1195  """
1196  maskedImage = ccdExposure.getMaskedImage()
1197 
1198  badAmp = False
1199 
1200  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1201  # comparison with current defects definition.
1202  if defects is not None:
1203  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1204 
1205  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1206  # association with pixels in current ccdExposure).
1207  if badAmp:
1208  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1209  afwImage.PARENT)
1210  maskView = dataView.getMask()
1211  maskView |= maskView.getPlaneBitMask("BAD")
1212  del maskView
1213  return badAmp
1214 
1215  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1216  # Saturation and suspect pixels can be masked now, though.
1217  limits = dict()
1218  if self.config.doSaturation and not badAmp:
1219  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1220  if self.config.doSuspect and not badAmp:
1221  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1222 
1223  for maskName, maskThreshold in limits.items():
1224  if not math.isnan(maskThreshold):
1225  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1226  isrFunctions.makeThresholdMask(
1227  maskedImage=dataView,
1228  threshold=maskThreshold,
1229  growFootprints=0,
1230  maskName=maskName
1231  )
1232 
1233  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1234  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1235  afwImage.PARENT)
1236  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1237  self.config.suspectMaskName])
1238  if numpy.all(maskView.getArray() & maskVal > 0):
1239  badAmp = True
1240 
1241  return badAmp
1242 
1243  def overscanCorrection(self, ccdExposure, amp):
1244  """Apply overscan correction in place.
1245 
1246  This method does initial pixel rejection of the overscan
1247  region. The overscan can also be optionally segmented to
1248  allow for discontinuous overscan responses to be fit
1249  separately. The actual overscan subtraction is performed by
1250  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1251  which is called here after the amplifier is preprocessed.
1252 
1253  Parameters
1254  ----------
1255  ccdExposure : `lsst.afw.image.Exposure`
1256  Exposure to have overscan correction performed.
1257  amp : `lsst.afw.table.AmpInfoCatalog`
1258  The amplifier to consider while correcting the overscan.
1259 
1260  Returns
1261  -------
1262  overscanResults : `lsst.pipe.base.Struct`
1263  Result struct with components:
1264  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1265  Value or fit subtracted from the amplifier image data.
1266  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1267  Value or fit subtracted from the overscan image data.
1268  - ``overscanImage`` : `lsst.afw.image.Image`
1269  Image of the overscan region with the overscan
1270  correction applied. This quantity is used to estimate
1271  the amplifier read noise empirically.
1272 
1273  Raises
1274  ------
1275  RuntimeError
1276  Raised if the ``amp`` does not contain raw pixel information.
1277 
1278  See Also
1279  --------
1280  lsst.ip.isr.isrFunctions.overscanCorrection
1281  """
1282  if not amp.getHasRawInfo():
1283  raise RuntimeError("This method must be executed on an amp with raw information.")
1284 
1285  if amp.getRawHorizontalOverscanBBox().isEmpty():
1286  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1287  return None
1288 
1289  # Construct views
1290  ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(),
1291  afwImage.PARENT)
1292  overscanImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
1293  amp.getRawHorizontalOverscanBBox(),
1294  afwImage.PARENT)
1295  overscanArray = overscanImage.getImage().getArray()
1296 
1297  statControl = afwMath.StatisticsControl()
1298  statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT"))
1299 
1300  # Determine the bounding boxes
1301  dataBBox = amp.getRawDataBBox()
1302  oscanBBox = amp.getRawHorizontalOverscanBBox()
1303  x0 = 0
1304  x1 = 0
1305 
1306  prescanBBox = amp.getRawPrescanBBox()
1307  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1308  x0 += self.config.overscanNumLeadingColumnsToSkip
1309  x1 -= self.config.overscanNumTrailingColumnsToSkip
1310  else:
1311  x0 += self.config.overscanNumTrailingColumnsToSkip
1312  x1 -= self.config.overscanNumLeadingColumnsToSkip
1313 
1314  # Determine if we need to work on subregions of the amplifier and overscan.
1315  imageBBoxes = []
1316  overscanBBoxes = []
1317 
1318  if ((self.config.overscanBiasJump and
1319  self.config.overscanBiasJumpLocation) and
1320  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1321  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1322  self.config.overscanBiasJumpDevices)):
1323  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1324  yLower = self.config.overscanBiasJumpLocation
1325  yUpper = dataBBox.getHeight() - yLower
1326  else:
1327  yUpper = self.config.overscanBiasJumpLocation
1328  yLower = dataBBox.getHeight() - yUpper
1329 
1330  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1331  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1332  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1333  afwGeom.Extent2I(x0, 0),
1334  afwGeom.Extent2I(oscanBBox.getWidth() + x1, yLower)))
1335 
1336  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1337  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1338 
1339  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(x0, yLower),
1340  afwGeom.Extent2I(oscanBBox.getWidth() + x1, yUpper)))
1341  else:
1342  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1343  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1344 
1345  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(x0, 0),
1346  afwGeom.Extent2I(oscanBBox.getWidth() + x1,
1347  oscanBBox.getHeight())))
1348 
1349  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1350  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1351  ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), imageBBox,
1352  afwImage.PARENT)
1353  overscanImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), overscanBBox,
1354  afwImage.PARENT)
1355 
1356  overscanArray = overscanImage.getImage().getArray()
1357  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.getMask().getArray(),
1358  overscanArray))
1359  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1360  overscanImage.getMask().getArray()[bad] = overscanImage.getMask().getPlaneBitMask("SAT")
1361 
1362  statControl = afwMath.StatisticsControl()
1363  statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT"))
1364 
1365  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1366  overscanImage=overscanImage,
1367  fitType=self.config.overscanFitType,
1368  order=self.config.overscanOrder,
1369  collapseRej=self.config.overscanNumSigmaClip,
1370  statControl=statControl,
1371  overscanIsInt=self.config.overscanIsInt
1372  )
1373 
1374  # Measure average overscan levels and record them in the metadata
1375  levelStat = afwMath.MEDIAN
1376  sigmaStat = afwMath.STDEVCLIP
1377 
1378  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1379  self.config.qa.flatness.nIter)
1380  metadata = ccdExposure.getMetadata()
1381  ampNum = amp.getName()
1382  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1383  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1384  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1385  else:
1386  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1387  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1388  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1389 
1390  return overscanResults
1391 
1392  def updateVariance(self, ampExposure, amp, overscanImage=None):
1393  """Set the variance plane using the amplifier gain and read noise
1394 
1395  The read noise is calculated from the ``overscanImage`` if the
1396  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1397  the value from the amplifier data is used.
1398 
1399  Parameters
1400  ----------
1401  ampExposure : `lsst.afw.image.Exposure`
1402  Exposure to process.
1403  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1404  Amplifier detector data.
1405  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1406  Image of overscan, required only for empirical read noise.
1407 
1408  See also
1409  --------
1410  lsst.ip.isr.isrFunctions.updateVariance
1411  """
1412  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1413  gain = amp.getGain()
1414 
1415  if math.isnan(gain):
1416  gain = 1.0
1417  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1418  elif gain <= 0:
1419  patchedGain = 1.0
1420  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1421  (amp.getName(), gain, patchedGain))
1422  gain = patchedGain
1423 
1424  if self.config.doEmpiricalReadNoise and overscanImage is None:
1425  self.log.info("Overscan is none for EmpiricalReadNoise")
1426 
1427  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1428  stats = afwMath.StatisticsControl()
1429  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1430  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1431  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1432  else:
1433  readNoise = amp.getReadNoise()
1434 
1435  isrFunctions.updateVariance(
1436  maskedImage=ampExposure.getMaskedImage(),
1437  gain=gain,
1438  readNoise=readNoise,
1439  )
1440 
1441  def darkCorrection(self, exposure, darkExposure, invert=False):
1442  """!Apply dark correction in place.
1443 
1444  Parameters
1445  ----------
1446  exposure : `lsst.afw.image.Exposure`
1447  Exposure to process.
1448  darkExposure : `lsst.afw.image.Exposure`
1449  Dark exposure of the same size as ``exposure``.
1450  invert : `Bool`, optional
1451  If True, re-add the dark to an already corrected image.
1452 
1453  Raises
1454  ------
1455  RuntimeError
1456  Raised if either ``exposure`` or ``darkExposure`` do not
1457  have their dark time defined.
1458 
1459  See Also
1460  --------
1461  lsst.ip.isr.isrFunctions.darkCorrection
1462  """
1463  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1464  if math.isnan(expScale):
1465  raise RuntimeError("Exposure darktime is NAN")
1466  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1467  if math.isnan(darkScale):
1468  raise RuntimeError("Dark calib darktime is NAN")
1469  isrFunctions.darkCorrection(
1470  maskedImage=exposure.getMaskedImage(),
1471  darkMaskedImage=darkExposure.getMaskedImage(),
1472  expScale=expScale,
1473  darkScale=darkScale,
1474  invert=invert,
1475  trimToFit=self.config.doTrimToMatchCalib
1476  )
1477 
1478  def doLinearize(self, detector):
1479  """!Check if linearization is needed for the detector cameraGeom.
1480 
1481  Checks config.doLinearize and the linearity type of the first
1482  amplifier.
1483 
1484  Parameters
1485  ----------
1486  detector : `lsst.afw.cameraGeom.Detector`
1487  Detector to get linearity type from.
1488 
1489  Returns
1490  -------
1491  doLinearize : `Bool`
1492  If True, linearization should be performed.
1493  """
1494  return self.config.doLinearize and \
1495  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1496 
1497  def flatCorrection(self, exposure, flatExposure, invert=False):
1498  """!Apply flat correction in place.
1499 
1500  Parameters
1501  ----------
1502  exposure : `lsst.afw.image.Exposure`
1503  Exposure to process.
1504  flatExposure : `lsst.afw.image.Exposure`
1505  Flat exposure of the same size as ``exposure``.
1506  invert : `Bool`, optional
1507  If True, unflatten an already flattened image.
1508 
1509  See Also
1510  --------
1511  lsst.ip.isr.isrFunctions.flatCorrection
1512  """
1513  isrFunctions.flatCorrection(
1514  maskedImage=exposure.getMaskedImage(),
1515  flatMaskedImage=flatExposure.getMaskedImage(),
1516  scalingType=self.config.flatScalingType,
1517  userScale=self.config.flatUserScale,
1518  invert=invert,
1519  trimToFit=self.config.doTrimToMatchCalib
1520  )
1521 
1522  def saturationDetection(self, exposure, amp):
1523  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1524 
1525  Parameters
1526  ----------
1527  exposure : `lsst.afw.image.Exposure`
1528  Exposure to process. Only the amplifier DataSec is processed.
1529  amp : `lsst.afw.table.AmpInfoCatalog`
1530  Amplifier detector data.
1531 
1532  See Also
1533  --------
1534  lsst.ip.isr.isrFunctions.makeThresholdMask
1535  """
1536  if not math.isnan(amp.getSaturation()):
1537  maskedImage = exposure.getMaskedImage()
1538  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1539  isrFunctions.makeThresholdMask(
1540  maskedImage=dataView,
1541  threshold=amp.getSaturation(),
1542  growFootprints=0,
1543  maskName=self.config.saturatedMaskName,
1544  )
1545 
1546  def saturationInterpolation(self, ccdExposure):
1547  """!Interpolate over saturated pixels, in place.
1548 
1549  This method should be called after `saturationDetection`, to
1550  ensure that the saturated pixels have been identified in the
1551  SAT mask. It should also be called after `assembleCcd`, since
1552  saturated regions may cross amplifier boundaries.
1553 
1554  Parameters
1555  ----------
1556  exposure : `lsst.afw.image.Exposure`
1557  Exposure to process.
1558 
1559  See Also
1560  --------
1561  lsst.ip.isr.isrTask.saturationDetection
1562  lsst.ip.isr.isrFunctions.interpolateFromMask
1563  """
1564  isrFunctions.interpolateFromMask(
1565  maskedImage=ccdExposure.getMaskedImage(),
1566  fwhm=self.config.fwhm,
1567  growFootprints=self.config.growSaturationFootprintSize,
1568  maskName=self.config.saturatedMaskName,
1569  )
1570 
1571  def suspectDetection(self, exposure, amp):
1572  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1573 
1574  Parameters
1575  ----------
1576  exposure : `lsst.afw.image.Exposure`
1577  Exposure to process. Only the amplifier DataSec is processed.
1578  amp : `lsst.afw.table.AmpInfoCatalog`
1579  Amplifier detector data.
1580 
1581  See Also
1582  --------
1583  lsst.ip.isr.isrFunctions.makeThresholdMask
1584 
1585  Notes
1586  -----
1587  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1588  This is intended to indicate pixels that may be affected by unknown systematics;
1589  for example if non-linearity corrections above a certain level are unstable
1590  then that would be a useful value for suspectLevel. A value of `nan` indicates
1591  that no such level exists and no pixels are to be masked as suspicious.
1592  """
1593  suspectLevel = amp.getSuspectLevel()
1594  if math.isnan(suspectLevel):
1595  return
1596 
1597  maskedImage = exposure.getMaskedImage()
1598  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1599  isrFunctions.makeThresholdMask(
1600  maskedImage=dataView,
1601  threshold=suspectLevel,
1602  growFootprints=0,
1603  maskName=self.config.suspectMaskName,
1604  )
1605 
1606  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1607  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1608 
1609  Parameters
1610  ----------
1611  ccdExposure : `lsst.afw.image.Exposure`
1612  Exposure to process.
1613  defectBaseList : `List`
1614  List of defects to mask and interpolate.
1615 
1616  Notes
1617  -----
1618  Call this after CCD assembly, since defects may cross amplifier boundaries.
1619  """
1620  maskedImage = ccdExposure.getMaskedImage()
1621  defectList = []
1622  for d in defectBaseList:
1623  bbox = d.getBBox()
1624  nd = measAlg.Defect(bbox)
1625  defectList.append(nd)
1626  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1627  isrFunctions.interpolateDefectList(
1628  maskedImage=maskedImage,
1629  defectList=defectList,
1630  fwhm=self.config.fwhm,
1631  )
1632 
1633  if self.config.numEdgeSuspect > 0:
1634  goodBBox = maskedImage.getBBox()
1635  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1636  goodBBox.grow(-self.config.numEdgeSuspect)
1637  # Mask pixels outside goodBBox as SUSPECT
1638  SourceDetectionTask.setEdgeBits(
1639  maskedImage,
1640  goodBBox,
1641  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1642  )
1643 
1644  def maskAndInterpNan(self, exposure):
1645  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1646 
1647  Parameters
1648  ----------
1649  exposure : `lsst.afw.image.Exposure`
1650  Exposure to process.
1651 
1652  Notes
1653  -----
1654  We mask and interpolate over all NaNs, including those
1655  that are masked with other bits (because those may or may
1656  not be interpolated over later, and we want to remove all
1657  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1658  is used to preserve the historical name.
1659  """
1660  maskedImage = exposure.getMaskedImage()
1661 
1662  # Find and mask NaNs
1663  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1664  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1665  numNans = maskNans(maskedImage, maskVal)
1666  self.metadata.set("NUMNANS", numNans)
1667 
1668  # Interpolate over these previously-unmasked NaNs
1669  if numNans > 0:
1670  self.log.warn("There were %i unmasked NaNs", numNans)
1671  nanDefectList = isrFunctions.getDefectListFromMask(
1672  maskedImage=maskedImage,
1673  maskName='UNMASKEDNAN',
1674  )
1675  isrFunctions.interpolateDefectList(
1676  maskedImage=exposure.getMaskedImage(),
1677  defectList=nanDefectList,
1678  fwhm=self.config.fwhm,
1679  )
1680 
1681  def measureBackground(self, exposure, IsrQaConfig=None):
1682  """Measure the image background in subgrids, for quality control purposes.
1683 
1684  Parameters
1685  ----------
1686  exposure : `lsst.afw.image.Exposure`
1687  Exposure to process.
1688  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
1689  Configuration object containing parameters on which background
1690  statistics and subgrids to use.
1691  """
1692  if IsrQaConfig is not None:
1693  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1694  IsrQaConfig.flatness.nIter)
1695  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
1696  statsControl.setAndMask(maskVal)
1697  maskedImage = exposure.getMaskedImage()
1698  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1699  skyLevel = stats.getValue(afwMath.MEDIAN)
1700  skySigma = stats.getValue(afwMath.STDEVCLIP)
1701  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1702  metadata = exposure.getMetadata()
1703  metadata.set('SKYLEVEL', skyLevel)
1704  metadata.set('SKYSIGMA', skySigma)
1705 
1706  # calcluating flatlevel over the subgrids
1707  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
1708  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1709  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
1710  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
1711  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
1712  skyLevels = numpy.zeros((nX, nY))
1713 
1714  for j in range(nY):
1715  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
1716  for i in range(nX):
1717  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
1718 
1719  xLLC = xc - meshXHalf
1720  yLLC = yc - meshYHalf
1721  xURC = xc + meshXHalf - 1
1722  yURC = yc + meshYHalf - 1
1723 
1724  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
1725  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
1726 
1727  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
1728 
1729  good = numpy.where(numpy.isfinite(skyLevels))
1730  skyMedian = numpy.median(skyLevels[good])
1731  flatness = (skyLevels[good] - skyMedian) / skyMedian
1732  flatness_rms = numpy.std(flatness)
1733  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
1734 
1735  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
1736  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
1737  (nX, nY, flatness_pp, flatness_rms))
1738 
1739  metadata.set('FLATNESS_PP', float(flatness_pp))
1740  metadata.set('FLATNESS_RMS', float(flatness_rms))
1741  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
1742  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
1743  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
1744 
1745  def roughZeroPoint(self, exposure):
1746  """Set an approximate magnitude zero point for the exposure.
1747 
1748  Parameters
1749  ----------
1750  exposure : `lsst.afw.image.Exposure`
1751  Exposure to process.
1752  """
1753  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
1754  if filterName in self.config.fluxMag0T1:
1755  fluxMag0 = self.config.fluxMag0T1[filterName]
1756  else:
1757  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
1758  fluxMag0 = self.config.defaultFluxMag0T1
1759 
1760  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
1761  if not expTime > 0: # handle NaN as well as <= 0
1762  self.log.warn("Non-positive exposure time; skipping rough zero point")
1763  return
1764 
1765  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
1766  exposure.getCalib().setFluxMag0(fluxMag0*expTime)
1767 
1768  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
1769  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
1770 
1771  Parameters
1772  ----------
1773  ccdExposure : `lsst.afw.image.Exposure`
1774  Exposure to process.
1775  fpPolygon : `lsst.afw.geom.Polygon`
1776  Polygon in focal plane coordinates.
1777  """
1778  # Get ccd corners in focal plane coordinates
1779  ccd = ccdExposure.getDetector()
1780  fpCorners = ccd.getCorners(FOCAL_PLANE)
1781  ccdPolygon = Polygon(fpCorners)
1782 
1783  # Get intersection of ccd corners with fpPolygon
1784  intersect = ccdPolygon.intersectionSingle(fpPolygon)
1785 
1786  # Transform back to pixel positions and build new polygon
1787  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
1788  validPolygon = Polygon(ccdPoints)
1789  ccdExposure.getInfo().setValidPolygon(validPolygon)
1790 
1791  @contextmanager
1792  def flatContext(self, exp, flat, dark=None):
1793  """Context manager that applies and removes flats and darks,
1794  if the task is configured to apply them.
1795 
1796  Parameters
1797  ----------
1798  exp : `lsst.afw.image.Exposure`
1799  Exposure to process.
1800  flat : `lsst.afw.image.Exposure`
1801  Flat exposure the same size as ``exp``.
1802  dark : `lsst.afw.image.Exposure`, optional
1803  Dark exposure the same size as ``exp``.
1804 
1805  Yields
1806  ------
1807  exp : `lsst.afw.image.Exposure`
1808  The flat and dark corrected exposure.
1809  """
1810  if self.config.doDark and dark is not None:
1811  self.darkCorrection(exp, dark)
1812  if self.config.doFlat:
1813  self.flatCorrection(exp, flat)
1814  try:
1815  yield exp
1816  finally:
1817  if self.config.doFlat:
1818  self.flatCorrection(exp, flat, invert=True)
1819  if self.config.doDark and dark is not None:
1820  self.darkCorrection(exp, dark, invert=True)
1821 
1822  def debugView(self, exposure, stepname):
1823  """Utility function to examine ISR exposure at different stages.
1824 
1825  Parameters
1826  ----------
1827  exposure : `lsst.afw.image.Exposure`
1828  Exposure to view.
1829  stepname : `str`
1830  State of processing to view.
1831  """
1832  frame = getDebugFrame(self._display, stepname)
1833  if frame:
1834  display = getDisplay(frame)
1835  display.scale('asinh', 'zscale')
1836  display.mtv(exposure)
1837 
1838 
1839 class FakeAmp(object):
1840  """A Detector-like object that supports returning gain and saturation level
1841 
1842  This is used when the input exposure does not have a detector.
1843 
1844  Parameters
1845  ----------
1846  exposure : `lsst.afw.image.Exposure`
1847  Exposure to generate a fake amplifier for.
1848  config : `lsst.ip.isr.isrTaskConfig`
1849  Configuration to apply to the fake amplifier.
1850  """
1851 
1852  def __init__(self, exposure, config):
1853  self._bbox = exposure.getBBox(afwImage.LOCAL)
1854  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
1855  self._gain = config.gain
1856  self._readNoise = config.readNoise
1857  self._saturation = config.saturation
1858 
1859  def getBBox(self):
1860  return self._bbox
1861 
1862  def getRawBBox(self):
1863  return self._bbox
1864 
1865  def getHasRawInfo(self):
1866  return True # but see getRawHorizontalOverscanBBox()
1867 
1869  return self._RawHorizontalOverscanBBox
1870 
1871  def getGain(self):
1872  return self._gain
1873 
1874  def getReadNoise(self):
1875  return self._readNoise
1876 
1877  def getSaturation(self):
1878  return self._saturation
1879 
1880  def getSuspectLevel(self):
1881  return float("NaN")
1882 
1883 
1884 class RunIsrConfig(pexConfig.Config):
1885  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
1886 
1887 
1888 class RunIsrTask(pipeBase.CmdLineTask):
1889  """Task to wrap the default IsrTask to allow it to be retargeted.
1890 
1891  The standard IsrTask can be called directly from a command line
1892  program, but doing so removes the ability of the task to be
1893  retargeted. As most cameras override some set of the IsrTask
1894  methods, this would remove those data-specific methods in the
1895  output post-ISR images. This wrapping class fixes the issue,
1896  allowing identical post-ISR images to be generated by both the
1897  processCcd and isrTask code.
1898  """
1899  ConfigClass = RunIsrConfig
1900  _DefaultName = "runIsr"
1901 
1902  def __init__(self, *args, **kwargs):
1903  super().__init__(*args, **kwargs)
1904  self.makeSubtask("isr")
1905 
1906  def runDataRef(self, dataRef):
1907  """
1908  Parameters
1909  ----------
1910  dataRef : `lsst.daf.persistence.ButlerDataRef`
1911  data reference of the detector data to be processed
1912 
1913  Returns
1914  -------
1915  result : `pipeBase.Struct`
1916  Result struct with component:
1917 
1918  - exposure : `lsst.afw.image.Exposure`
1919  Post-ISR processed exposure.
1920  """
1921  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1047
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:1681
def debugView(self, exposure, stepname)
Definition: isrTask.py:1822
def __init__(self, args, kwargs)
Definition: isrTask.py:577
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:587
def runDataRef(self, dataRef)
Definition: isrTask.py:1906
def __init__(self, args, kwargs)
Definition: isrTask.py:1902
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1644
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1546
def roughZeroPoint(self, exposure)
Definition: isrTask.py:1745
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:690
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:1868
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1243
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1139
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1497
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1098
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1441
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1478
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:1768
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1175
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:1792
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1392
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1571
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1606
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1522
def __init__(self, exposure, config)
Definition: isrTask.py:1852