lsst.ip.isr  16.0-25-g5e2e023+2
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defect
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "RunIsrTask"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="ExposureU",
78  dimensions=["Instrument", "Exposure", "Detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["Instrument", "ExposureRange"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["Instrument", "ExposureRange", "Detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["Instrument", "ExposureRange", "Detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["Instrument", "PhysicalFilter", "ExposureRange", "Detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["Instrument", "ExposureRange"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="Catalog",
120  dimensions=["Instrument", "ExposureRange", "Detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["Instrument", "ExposureRange"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["Instrument", "PhysicalFilter", "ExposureRange"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["Instrument", "ExposureRange", "Detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["Instrument"],
149  )
150 
151  # output datasets
152  outputExposure = pipeBase.OutputDatasetField(
153  doc="Output ISR processed exposure.",
154  name="postISRCCD",
155  scalar=True,
156  storageClass="ExposureF",
157  dimensions=["Instrument", "Visit", "Detector"],
158  )
159  outputOssThumbnail = pipeBase.OutputDatasetField(
160  doc="Output Overscan-subtracted thumbnail image.",
161  name="OssThumb",
162  scalar=True,
163  storageClass="Thumbnail",
164  dimensions=["Instrument", "Visit", "Detector"],
165  )
166  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167  doc="Output flat-corrected thumbnail image.",
168  name="FlattenedThumb",
169  scalar=True,
170  storageClass="TextStorage",
171  dimensions=["Instrument", "Visit", "Detector"],
172  )
173 
174  quantum = pipeBase.QuantumConfig(
175  dimensions=["Visit", "Detector", "Instrument"],
176  )
177 
178 
179  datasetType = pexConfig.Field(
180  dtype=str,
181  doc="Dataset type for input data; users will typically leave this alone, "
182  "but camera-specific ISR tasks will override it",
183  default="raw",
184  )
185 
186  fallbackFilterName = pexConfig.Field(
187  dtype=str,
188  doc="Fallback default filter name for calibrations.",
189  optional=True
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
195  )
196  fwhm = pexConfig.Field(
197  dtype=float,
198  doc="FWHM of PSF in arcseconds.",
199  default=1.0,
200  )
201  qa = pexConfig.ConfigField(
202  dtype=isrQa.IsrQaConfig,
203  doc="QA related configuration options.",
204  )
205 
206  # Image conversion configuration
207  doConvertIntToFloat = pexConfig.Field(
208  dtype=bool,
209  doc="Convert integer raw images to floating point values?",
210  default=True,
211  )
212 
213  # Saturated pixel handling.
214  doSaturation = pexConfig.Field(
215  dtype=bool,
216  doc="Mask saturated pixels? NB: this is totally independent of the"
217  " interpolation option - this is ONLY setting the bits in the mask."
218  " To have them interpolated make sure doSaturationInterpolation=True",
219  default=True,
220  )
221  saturatedMaskName = pexConfig.Field(
222  dtype=str,
223  doc="Name of mask plane to use in saturation detection and interpolation",
224  default="SAT",
225  )
226  saturation = pexConfig.Field(
227  dtype=float,
228  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229  default=float("NaN"),
230  )
231  growSaturationFootprintSize = pexConfig.Field(
232  dtype=int,
233  doc="Number of pixels by which to grow the saturation footprints",
234  default=1,
235  )
236 
237  # Suspect pixel handling.
238  doSuspect = pexConfig.Field(
239  dtype=bool,
240  doc="Mask suspect pixels?",
241  default=True,
242  )
243  suspectMaskName = pexConfig.Field(
244  dtype=str,
245  doc="Name of mask plane to use for suspect pixels",
246  default="SUSPECT",
247  )
248  numEdgeSuspect = pexConfig.Field(
249  dtype=int,
250  doc="Number of edge pixels to be flagged as untrustworthy.",
251  default=0,
252  )
253 
254  # Initial masking options.
255  doSetBadRegions = pexConfig.Field(
256  dtype=bool,
257  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
258  default=True,
259  )
260  badStatistic = pexConfig.ChoiceField(
261  dtype=str,
262  doc="How to estimate the average value for BAD regions.",
263  default='MEANCLIP',
264  allowed={
265  "MEANCLIP": "Correct using the (clipped) mean of good data",
266  "MEDIAN": "Correct using the median of the good data",
267  },
268  )
269 
270  # Overscan subtraction configuration.
271  doOverscan = pexConfig.Field(
272  dtype=bool,
273  doc="Do overscan subtraction?",
274  default=True,
275  )
276  overscanFitType = pexConfig.ChoiceField(
277  dtype=str,
278  doc="The method for fitting the overscan bias level.",
279  default='MEDIAN',
280  allowed={
281  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
282  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
283  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
284  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
285  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
286  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
287  "MEAN": "Correct using the mean of the overscan region",
288  "MEANCLIP": "Correct using a clipped mean of the overscan region",
289  "MEDIAN": "Correct using the median of the overscan region",
290  },
291  )
292  overscanOrder = pexConfig.Field(
293  dtype=int,
294  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
295  "or number of spline knots if overscan fit type is a spline."),
296  default=1,
297  )
298  overscanNumSigmaClip = pexConfig.Field(
299  dtype=float,
300  doc="Rejection threshold (sigma) for collapsing overscan before fit",
301  default=3.0,
302  )
303  overscanIsInt = pexConfig.Field(
304  dtype=bool,
305  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
306  default=True,
307  )
308  overscanNumLeadingColumnsToSkip = pexConfig.Field(
309  dtype=int,
310  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
311  default=0,
312  )
313  overscanNumTrailingColumnsToSkip = pexConfig.Field(
314  dtype=int,
315  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
316  default=0,
317  )
318  overscanMaxDev = pexConfig.Field(
319  dtype=float,
320  doc="Maximum deviation from the median for overscan",
321  default=1000.0, check=lambda x: x > 0
322  )
323  overscanBiasJump = pexConfig.Field(
324  dtype=bool,
325  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
326  default=False,
327  )
328  overscanBiasJumpKeyword = pexConfig.Field(
329  dtype=str,
330  doc="Header keyword containing information about devices.",
331  default="NO_SUCH_KEY",
332  )
333  overscanBiasJumpDevices = pexConfig.ListField(
334  dtype=str,
335  doc="List of devices that need piecewise overscan correction.",
336  default=(),
337  )
338  overscanBiasJumpLocation = pexConfig.Field(
339  dtype=int,
340  doc="Location of bias jump along y-axis.",
341  default=0,
342  )
343 
344  # Amplifier to CCD assembly configuration
345  doAssembleCcd = pexConfig.Field(
346  dtype=bool,
347  default=True,
348  doc="Assemble amp-level exposures into a ccd-level exposure?"
349  )
350  assembleCcd = pexConfig.ConfigurableField(
351  target=AssembleCcdTask,
352  doc="CCD assembly task",
353  )
354 
355  # General calibration configuration.
356  doAssembleIsrExposures = pexConfig.Field(
357  dtype=bool,
358  default=False,
359  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
360  )
361  doTrimToMatchCalib = pexConfig.Field(
362  dtype=bool,
363  default=False,
364  doc="Trim raw data to match calibration bounding boxes?"
365  )
366 
367  # Bias subtraction.
368  doBias = pexConfig.Field(
369  dtype=bool,
370  doc="Apply bias frame correction?",
371  default=True,
372  )
373  biasDataProductName = pexConfig.Field(
374  dtype=str,
375  doc="Name of the bias data product",
376  default="bias",
377  )
378 
379  # Variance construction
380  doVariance = pexConfig.Field(
381  dtype=bool,
382  doc="Calculate variance?",
383  default=True
384  )
385  gain = pexConfig.Field(
386  dtype=float,
387  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388  default=float("NaN"),
389  )
390  readNoise = pexConfig.Field(
391  dtype=float,
392  doc="The read noise to use if no Detector is present in the Exposure",
393  default=0.0,
394  )
395  doEmpiricalReadNoise = pexConfig.Field(
396  dtype=bool,
397  default=False,
398  doc="Calculate empirical read noise instead of value from AmpInfo data?"
399  )
400 
401  # Linearization.
402  doLinearize = pexConfig.Field(
403  dtype=bool,
404  doc="Correct for nonlinearity of the detector's response?",
405  default=True,
406  )
407 
408  # Crosstalk.
409  doCrosstalk = pexConfig.Field(
410  dtype=bool,
411  doc="Apply intra-CCD crosstalk correction?",
412  default=False,
413  )
414  doCrosstalkBeforeAssemble = pexConfig.Field(
415  dtype=bool,
416  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
417  default=True,
418  )
419  crosstalk = pexConfig.ConfigurableField(
420  target=CrosstalkTask,
421  doc="Intra-CCD crosstalk correction",
422  )
423 
424  # Masking option prior to brighter-fatter.
425  doWidenSaturationTrails = pexConfig.Field(
426  dtype=bool,
427  doc="Widen bleed trails based on their width?",
428  default=True
429  )
430 
431  # Brighter-Fatter correction.
432  doBrighterFatter = pexConfig.Field(
433  dtype=bool,
434  default=False,
435  doc="Apply the brighter fatter correction"
436  )
437  brighterFatterLevel = pexConfig.ChoiceField(
438  dtype=str,
439  default="DETECTOR",
440  doc="The level at which to correct for brighter-fatter.",
441  allowed={
442  "AMP": "Every amplifier treated separately.",
443  "DETECTOR": "One kernel per detector",
444  }
445  )
446  brighterFatterKernelFile = pexConfig.Field(
447  dtype=str,
448  default='',
449  doc="Kernel file used for the brighter fatter correction"
450  )
451  brighterFatterMaxIter = pexConfig.Field(
452  dtype=int,
453  default=10,
454  doc="Maximum number of iterations for the brighter fatter correction"
455  )
456  brighterFatterThreshold = pexConfig.Field(
457  dtype=float,
458  default=1000,
459  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
460  " absolute value of the difference between the current corrected image and the one"
461  " from the previous iteration summed over all the pixels."
462  )
463  brighterFatterApplyGain = pexConfig.Field(
464  dtype=bool,
465  default=True,
466  doc="Should the gain be applied when applying the brighter fatter correction?"
467  )
468 
469  # Defect and bad pixel correction options.
470  doDefect = pexConfig.Field(
471  dtype=bool,
472  doc="Apply correction for CCD defects, e.g. hot pixels?",
473  default=True,
474  )
475  doSaturationInterpolation = pexConfig.Field(
476  dtype=bool,
477  doc="Perform interpolation over pixels masked as saturated?"
478  " NB: This is independent of doSaturation; if that is False this plane"
479  " will likely be blank, resulting in a no-op here.",
480  default=True,
481  )
482  numEdgeSuspect = pexConfig.Field(
483  dtype=int,
484  doc="Number of edge pixels to be flagged as untrustworthy.",
485  default=0,
486  )
487 
488  # Dark subtraction.
489  doDark = pexConfig.Field(
490  dtype=bool,
491  doc="Apply dark frame correction?",
492  default=True,
493  )
494  darkDataProductName = pexConfig.Field(
495  dtype=str,
496  doc="Name of the dark data product",
497  default="dark",
498  )
499 
500  # Camera-specific stray light removal.
501  doStrayLight = pexConfig.Field(
502  dtype=bool,
503  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
504  default=False,
505  )
506  strayLight = pexConfig.ConfigurableField(
507  target=StrayLightTask,
508  doc="y-band stray light correction"
509  )
510 
511  # Flat correction.
512  doFlat = pexConfig.Field(
513  dtype=bool,
514  doc="Apply flat field correction?",
515  default=True,
516  )
517  flatDataProductName = pexConfig.Field(
518  dtype=str,
519  doc="Name of the flat data product",
520  default="flat",
521  )
522  flatScalingType = pexConfig.ChoiceField(
523  dtype=str,
524  doc="The method for scaling the flat on the fly.",
525  default='USER',
526  allowed={
527  "USER": "Scale by flatUserScale",
528  "MEAN": "Scale by the inverse of the mean",
529  "MEDIAN": "Scale by the inverse of the median",
530  },
531  )
532  flatUserScale = pexConfig.Field(
533  dtype=float,
534  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
535  default=1.0,
536  )
537  doTweakFlat = pexConfig.Field(
538  dtype=bool,
539  doc="Tweak flats to match observed amplifier ratios?",
540  default=False
541  )
542 
543  # Amplifier normalization based on gains instead of using flats configuration.
544  doApplyGains = pexConfig.Field(
545  dtype=bool,
546  doc="Correct the amplifiers for their gains instead of applying flat correction",
547  default=False,
548  )
549  normalizeGains = pexConfig.Field(
550  dtype=bool,
551  doc="Normalize all the amplifiers in each CCD to have the same median value.",
552  default=False,
553  )
554 
555  # Fringe correction.
556  doFringe = pexConfig.Field(
557  dtype=bool,
558  doc="Apply fringe correction?",
559  default=True,
560  )
561  fringe = pexConfig.ConfigurableField(
562  target=FringeTask,
563  doc="Fringe subtraction task",
564  )
565  fringeAfterFlat = pexConfig.Field(
566  dtype=bool,
567  doc="Do fringe subtraction after flat-fielding?",
568  default=True,
569  )
570 
571  # NAN pixel interpolation option.
572  doNanInterpAfterFlat = pexConfig.Field(
573  dtype=bool,
574  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
575  "also have to interpolate them before flat-fielding."),
576  default=False,
577  )
578 
579  # Distortion model application.
580  doAddDistortionModel = pexConfig.Field(
581  dtype=bool,
582  doc="Apply a distortion model based on camera geometry to the WCS?",
583  default=True,
584  )
585 
586  # Initial CCD-level background statistics options.
587  doMeasureBackground = pexConfig.Field(
588  dtype=bool,
589  doc="Measure the background level on the reduced image?",
590  default=False,
591  )
592 
593  # Camera-specific masking configuration.
594  doCameraSpecificMasking = pexConfig.Field(
595  dtype=bool,
596  doc="Mask camera-specific bad regions?",
597  default=False,
598  )
599  masking = pexConfig.ConfigurableField(
600  target=MaskingTask,
601  doc="Masking task."
602  )
603 
604  # Default photometric calibration options.
605  fluxMag0T1 = pexConfig.DictField(
606  keytype=str,
607  itemtype=float,
608  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
610  ))
611  )
612  defaultFluxMag0T1 = pexConfig.Field(
613  dtype=float,
614  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
615  default=pow(10.0, 0.4*28.0)
616  )
617 
618  # Vignette correction configuration.
619  doVignette = pexConfig.Field(
620  dtype=bool,
621  doc="Apply vignetting parameters?",
622  default=False,
623  )
624  vignette = pexConfig.ConfigurableField(
625  target=VignetteTask,
626  doc="Vignetting task.",
627  )
628 
629  # Transmission curve configuration.
630  doAttachTransmissionCurve = pexConfig.Field(
631  dtype=bool,
632  default=False,
633  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
634  )
635  doUseOpticsTransmission = pexConfig.Field(
636  dtype=bool,
637  default=True,
638  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
639  )
640  doUseFilterTransmission = pexConfig.Field(
641  dtype=bool,
642  default=True,
643  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
644  )
645  doUseSensorTransmission = pexConfig.Field(
646  dtype=bool,
647  default=True,
648  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
649  )
650  doUseAtmosphereTransmission = pexConfig.Field(
651  dtype=bool,
652  default=True,
653  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
654  )
655 
656  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
657  doWrite = pexConfig.Field(
658  dtype=bool,
659  doc="Persist postISRCCD?",
660  default=True,
661  )
662 
663  def validate(self):
664  super().validate()
665  if self.doFlat and self.doApplyGains:
666  raise ValueError("You may not specify both doFlat and doApplyGains")
667 
668 
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670  r"""Apply common instrument signature correction algorithms to a raw frame.
671 
672  The process for correcting imaging data is very similar from
673  camera to camera. This task provides a vanilla implementation of
674  doing these corrections, including the ability to turn certain
675  corrections off if they are not needed. The inputs to the primary
676  method, `run()`, are a raw exposure to be corrected and the
677  calibration data products. The raw input is a single chip sized
678  mosaic of all amps including overscans and other non-science
679  pixels. The method `runDataRef()` identifies and defines the
680  calibration data products, and is intended for use by a
681  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
682  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
683  subclassed for different camera, although the most camera specific
684  methods have been split into subtasks that can be redirected
685  appropriately.
686 
687  The __init__ method sets up the subtasks for ISR processing, using
688  the defaults from `lsst.ip.isr`.
689 
690  Parameters
691  ----------
692  args : `list`
693  Positional arguments passed to the Task constructor. None used at this time.
694  kwargs : `dict`, optional
695  Keyword arguments passed on to the Task constructor. None used at this time.
696  """
697  ConfigClass = IsrTaskConfig
698  _DefaultName = "isr"
699 
700  def __init__(self, **kwargs):
701  super().__init__(**kwargs)
702  self.makeSubtask("assembleCcd")
703  self.makeSubtask("crosstalk")
704  self.makeSubtask("strayLight")
705  self.makeSubtask("fringe")
706  self.makeSubtask("masking")
707  self.makeSubtask("vignette")
708 
709  @classmethod
710  def getInputDatasetTypes(cls, config):
711  inputTypeDict = super().getInputDatasetTypes(config)
712 
713  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
714  # need because the configuration tells us we will not be bothering with the
715  # correction that uses that IDT.
716  if config.doBias is not True:
717  inputTypeDict.pop("bias", None)
718  if config.doLinearize is not True:
719  inputTypeDict.pop("linearizer", None)
720  if config.doCrosstalk is not True:
721  inputTypeDict.pop("crosstalkSources", None)
722  if config.doBrighterFatter is not True:
723  inputTypeDict.pop("bfKernel", None)
724  if config.doDefect is not True:
725  inputTypeDict.pop("defects", None)
726  if config.doDark is not True:
727  inputTypeDict.pop("dark", None)
728  if config.doFlat is not True:
729  inputTypeDict.pop("flat", None)
730  if config.doAttachTransmissionCurve is not True:
731  inputTypeDict.pop("opticsTransmission", None)
732  inputTypeDict.pop("filterTransmission", None)
733  inputTypeDict.pop("sensorTransmission", None)
734  inputTypeDict.pop("atmosphereTransmission", None)
735  if config.doUseOpticsTransmission is not True:
736  inputTypeDict.pop("opticsTransmission", None)
737  if config.doUseFilterTransmission is not True:
738  inputTypeDict.pop("filterTransmission", None)
739  if config.doUseSensorTransmission is not True:
740  inputTypeDict.pop("sensorTransmission", None)
741  if config.doUseAtmosphereTransmission is not True:
742  inputTypeDict.pop("atmosphereTransmission", None)
743 
744  return inputTypeDict
745 
746  @classmethod
747  def getOutputDatasetTypes(cls, config):
748  outputTypeDict = super().getOutputDatasetTypes(config)
749 
750  if config.qa.doThumbnailOss is not True:
751  outputTypeDict.pop("outputOssThumbnail", None)
752  if config.qa.doThumbnailFlattened is not True:
753  outputTypeDict.pop("outputFlattenedThumbnail", None)
754  if config.doWrite is not True:
755  outputTypeDict.pop("outputExposure", None)
756 
757  return outputTypeDict
758 
759  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
760  try:
761  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
762  except Exception as e:
763  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
764 
765  inputData['isGen3'] = True
766 
767  if self.config.doLinearize is True:
768  if 'linearizer' not in inputData.keys():
769  detector = inputData['camera'][inputData['detectorNum']]
770  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
771  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
772 
773  if inputData['defects'] is not None:
774  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
775  # masking expects a list of defects defined by their bounding box
776  defectList = []
777 
778  for r in inputData['defects']:
779  bbox = afwGeom.BoxI(afwGeom.PointI(r.get("x0"), r.get("y0")),
780  afwGeom.ExtentI(r.get("width"), r.get("height")))
781  defectList.append(Defect(bbox))
782 
783  inputData['defects'] = defectList
784 
785  # Broken: DM-17169
786  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
787  # fixed for non-HSC cameras in the future.
788  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
789  # if self.config.doCrosstalk else None)
790 
791  # Broken: DM-17152
792  # Fringes are not tested to be handled correctly by Gen3 butler.
793  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
794  # assembler=self.assembleCcd
795  # if self.config.doAssembleIsrExposures else None)
796  # if self.config.doFringe and
797  # self.fringe.checkFilter(inputData['ccdExposure'])
798  # else pipeBase.Struct(fringes=None))
799 
800  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
801 
802  def makeDatasetType(self, dsConfig):
803  return super().makeDatasetType(dsConfig)
804 
805  def readIsrData(self, dataRef, rawExposure):
806  """!Retrieve necessary frames for instrument signature removal.
807 
808  Pre-fetching all required ISR data products limits the IO
809  required by the ISR. Any conflict between the calibration data
810  available and that needed for ISR is also detected prior to
811  doing processing, allowing it to fail quickly.
812 
813  Parameters
814  ----------
815  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
816  Butler reference of the detector data to be processed
817  rawExposure : `afw.image.Exposure`
818  The raw exposure that will later be corrected with the
819  retrieved calibration data; should not be modified in this
820  method.
821 
822  Returns
823  -------
824  result : `lsst.pipe.base.Struct`
825  Result struct with components (which may be `None`):
826  - ``bias``: bias calibration frame (`afw.image.Exposure`)
827  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
828  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
829  - ``dark``: dark calibration frame (`afw.image.Exposure`)
830  - ``flat``: flat calibration frame (`afw.image.Exposure`)
831  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
832  - ``defects``: list of defects (`list`)
833  - ``fringes``: `lsst.pipe.base.Struct` with components:
834  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
835  - ``seed``: random seed derived from the ccdExposureId for random
836  number generator (`uint32`)
837  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
838  A ``TransmissionCurve`` that represents the throughput of the optics,
839  to be evaluated in focal-plane coordinates.
840  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
841  A ``TransmissionCurve`` that represents the throughput of the filter
842  itself, to be evaluated in focal-plane coordinates.
843  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
844  A ``TransmissionCurve`` that represents the throughput of the sensor
845  itself, to be evaluated in post-assembly trimmed detector coordinates.
846  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
847  A ``TransmissionCurve`` that represents the throughput of the
848  atmosphere, assumed to be spatially constant.
849 
850  """
851  ccd = rawExposure.getDetector()
852  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
853  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
854  if self.config.doBias else None)
855  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
856  linearizer = (dataRef.get("linearizer", immediate=True)
857  if self.doLinearize(ccd) else None)
858  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
859  if self.config.doCrosstalk else None)
860  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
861  if self.config.doDark else None)
862  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
863  if self.config.doFlat else None)
864  brighterFatterKernel = (dataRef.get("bfKernel")
865  if self.config.doBrighterFatter else None)
866  defectList = (dataRef.get("defects")
867  if self.config.doDefect else None)
868  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
869  if self.config.doAssembleIsrExposures else None)
870  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
871  else pipeBase.Struct(fringes=None))
872 
873  if self.config.doAttachTransmissionCurve:
874  opticsTransmission = (dataRef.get("transmission_optics")
875  if self.config.doUseOpticsTransmission else None)
876  filterTransmission = (dataRef.get("transmission_filter")
877  if self.config.doUseFilterTransmission else None)
878  sensorTransmission = (dataRef.get("transmission_sensor")
879  if self.config.doUseSensorTransmission else None)
880  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
881  if self.config.doUseAtmosphereTransmission else None)
882  else:
883  opticsTransmission = None
884  filterTransmission = None
885  sensorTransmission = None
886  atmosphereTransmission = None
887 
888  # Struct should include only kwargs to run()
889  return pipeBase.Struct(bias=biasExposure,
890  linearizer=linearizer,
891  crosstalkSources=crosstalkSources,
892  dark=darkExposure,
893  flat=flatExposure,
894  bfKernel=brighterFatterKernel,
895  defects=defectList,
896  fringes=fringeStruct,
897  opticsTransmission=opticsTransmission,
898  filterTransmission=filterTransmission,
899  sensorTransmission=sensorTransmission,
900  atmosphereTransmission=atmosphereTransmission,
901  )
902 
903  @pipeBase.timeMethod
904  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
905  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
906  opticsTransmission=None, filterTransmission=None,
907  sensorTransmission=None, atmosphereTransmission=None,
908  detectorNum=None, isGen3=False
909  ):
910  """!Perform instrument signature removal on an exposure.
911 
912  Steps included in the ISR processing, in order performed, are:
913  - saturation and suspect pixel masking
914  - overscan subtraction
915  - CCD assembly of individual amplifiers
916  - bias subtraction
917  - variance image construction
918  - linearization of non-linear response
919  - crosstalk masking
920  - brighter-fatter correction
921  - dark subtraction
922  - fringe correction
923  - stray light subtraction
924  - flat correction
925  - masking of known defects and camera specific features
926  - vignette calculation
927  - appending transmission curve and distortion model
928 
929  Parameters
930  ----------
931  ccdExposure : `lsst.afw.image.Exposure`
932  The raw exposure that is to be run through ISR. The
933  exposure is modified by this method.
934  camera : `lsst.afw.cameraGeom.Camera`, optional
935  The camera geometry for this exposure. Used to select the
936  distortion model appropriate for this data.
937  bias : `lsst.afw.image.Exposure`, optional
938  Bias calibration frame.
939  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
940  Functor for linearization.
941  crosstalkSources : `list`, optional
942  List of possible crosstalk sources.
943  dark : `lsst.afw.image.Exposure`, optional
944  Dark calibration frame.
945  flat : `lsst.afw.image.Exposure`, optional
946  Flat calibration frame.
947  bfKernel : `numpy.ndarray`, optional
948  Brighter-fatter kernel.
949  defects : `list`, optional
950  List of defects.
951  fringes : `lsst.pipe.base.Struct`, optional
952  Struct containing the fringe correction data, with
953  elements:
954  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
955  - ``seed``: random seed derived from the ccdExposureId for random
956  number generator (`uint32`)
957  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
958  A ``TransmissionCurve`` that represents the throughput of the optics,
959  to be evaluated in focal-plane coordinates.
960  filterTransmission : `lsst.afw.image.TransmissionCurve`
961  A ``TransmissionCurve`` that represents the throughput of the filter
962  itself, to be evaluated in focal-plane coordinates.
963  sensorTransmission : `lsst.afw.image.TransmissionCurve`
964  A ``TransmissionCurve`` that represents the throughput of the sensor
965  itself, to be evaluated in post-assembly trimmed detector coordinates.
966  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
967  A ``TransmissionCurve`` that represents the throughput of the
968  atmosphere, assumed to be spatially constant.
969  detectorNum : `int`, optional
970  The integer number for the detector to process.
971  isGen3 : bool, optional
972  Flag this call to run() as using the Gen3 butler environment.
973 
974  Returns
975  -------
976  result : `lsst.pipe.base.Struct`
977  Result struct with component:
978  - ``exposure`` : `afw.image.Exposure`
979  The fully ISR corrected exposure.
980  - ``outputExposure`` : `afw.image.Exposure`
981  An alias for `exposure`
982  - ``ossThumb`` : `numpy.ndarray`
983  Thumbnail image of the exposure after overscan subtraction.
984  - ``flattenedThumb`` : `numpy.ndarray`
985  Thumbnail image of the exposure after flat-field correction.
986 
987  Raises
988  ------
989  RuntimeError
990  Raised if a configuration option is set to True, but the
991  required calibration data has not been specified.
992 
993  Notes
994  -----
995  The current processed exposure can be viewed by setting the
996  appropriate lsstDebug entries in the `debug.display`
997  dictionary. The names of these entries correspond to some of
998  the IsrTaskConfig Boolean options, with the value denoting the
999  frame to use. The exposure is shown inside the matching
1000  option check and after the processing of that step has
1001  finished. The steps with debug points are:
1002 
1003  doAssembleCcd
1004  doBias
1005  doCrosstalk
1006  doBrighterFatter
1007  doDark
1008  doFringe
1009  doStrayLight
1010  doFlat
1011 
1012  In addition, setting the "postISRCCD" entry displays the
1013  exposure after all ISR processing has finished.
1014 
1015  """
1016 
1017  if isGen3 is True:
1018  # Gen3 currently cannot automatically do configuration overrides.
1019  # DM-15257 looks to discuss this issue.
1020 
1021  self.config.doFringe = False
1022 
1023  # Configure input exposures;
1024  if detectorNum is None:
1025  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1026 
1027  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1028  bias = self.ensureExposure(bias, camera, detectorNum)
1029  dark = self.ensureExposure(dark, camera, detectorNum)
1030  flat = self.ensureExposure(flat, camera, detectorNum)
1031  else:
1032  if isinstance(ccdExposure, ButlerDataRef):
1033  return self.runDataRef(ccdExposure)
1034 
1035  ccd = ccdExposure.getDetector()
1036 
1037  if not ccd:
1038  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1039  ccd = [FakeAmp(ccdExposure, self.config)]
1040 
1041  # Validate Input
1042  if self.config.doBias and bias is None:
1043  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1044  if self.doLinearize(ccd) and linearizer is None:
1045  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1046  if self.config.doBrighterFatter and bfKernel is None:
1047  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1048  if self.config.doDark and dark is None:
1049  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1050  if fringes is None:
1051  fringes = pipeBase.Struct(fringes=None)
1052  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1053  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1054  if self.config.doFlat and flat is None:
1055  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1056  if self.config.doDefect and defects is None:
1057  raise RuntimeError("Must supply defects if config.doDefect=True.")
1058  if self.config.doAddDistortionModel and camera is None:
1059  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1060 
1061  # Begin ISR processing.
1062  if self.config.doConvertIntToFloat:
1063  self.log.info("Converting exposure to floating point values")
1064  ccdExposure = self.convertIntToFloat(ccdExposure)
1065 
1066  # Amplifier level processing.
1067  overscans = []
1068  for amp in ccd:
1069  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1070  if ccdExposure.getBBox().contains(amp.getBBox()):
1071  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1072  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1073 
1074  if self.config.doOverscan and not badAmp:
1075  # Overscan correction on amp-by-amp basis.
1076  overscanResults = self.overscanCorrection(ccdExposure, amp)
1077  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1078  if self.config.qa is not None and self.config.qa.saveStats is True:
1079  if isinstance(overscanResults.overscanFit, float):
1080  qaMedian = overscanResults.overscanFit
1081  qaStdev = float("NaN")
1082  else:
1083  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1084  afwMath.MEDIAN | afwMath.STDEVCLIP)
1085  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1086  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1087 
1088  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1089  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1090  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1091  (amp.getName(), qaMedian, qaStdev))
1092  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1093  else:
1094  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1095  overscanResults = None
1096 
1097  overscans.append(overscanResults if overscanResults is not None else None)
1098  else:
1099  self.log.info("Skipped OSCAN")
1100 
1101  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1102  self.log.info("Applying crosstalk correction.")
1103  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1104  self.debugView(ccdExposure, "doCrosstalk")
1105 
1106  if self.config.doAssembleCcd:
1107  self.log.info("Assembling CCD from amplifiers")
1108  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1109 
1110  if self.config.expectWcs and not ccdExposure.getWcs():
1111  self.log.warn("No WCS found in input exposure")
1112  self.debugView(ccdExposure, "doAssembleCcd")
1113 
1114  ossThumb = None
1115  if self.config.qa.doThumbnailOss:
1116  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1117 
1118  if self.config.doBias:
1119  self.log.info("Applying bias correction.")
1120  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1121  trimToFit=self.config.doTrimToMatchCalib)
1122  self.debugView(ccdExposure, "doBias")
1123 
1124  if self.config.doVariance:
1125  for amp, overscanResults in zip(ccd, overscans):
1126  if ccdExposure.getBBox().contains(amp.getBBox()):
1127  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1128  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1129  if overscanResults is not None:
1130  self.updateVariance(ampExposure, amp,
1131  overscanImage=overscanResults.overscanImage)
1132  else:
1133  self.updateVariance(ampExposure, amp,
1134  overscanImage=None)
1135  if self.config.qa is not None and self.config.qa.saveStats is True:
1136  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1137  afwMath.MEDIAN | afwMath.STDEVCLIP)
1138  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1139  qaStats.getValue(afwMath.MEDIAN))
1140  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1141  qaStats.getValue(afwMath.STDEVCLIP))
1142  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1143  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1144  qaStats.getValue(afwMath.STDEVCLIP)))
1145 
1146  if self.doLinearize(ccd):
1147  self.log.info("Applying linearizer.")
1148  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1149 
1150  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1151  self.log.info("Applying crosstalk correction.")
1152  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1153  self.debugView(ccdExposure, "doCrosstalk")
1154 
1155  if self.config.doWidenSaturationTrails:
1156  self.log.info("Widening saturation trails.")
1157  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1158 
1159  interpolationDone = False
1160  if self.config.doBrighterFatter:
1161  # We need to apply flats and darks before we can interpolate, and we
1162  # need to interpolate before we do B-F, but we do B-F without the
1163  # flats and darks applied so we can work in units of electrons or holes.
1164  # This context manager applies and then removes the darks and flats.
1165  with self.flatContext(ccdExposure, flat, dark):
1166  if self.config.doDefect:
1167  self.maskAndInterpDefect(ccdExposure, defects)
1168 
1169  if self.config.doSaturationInterpolation:
1170  self.saturationInterpolation(ccdExposure)
1171 
1172  self.maskAndInterpNan(ccdExposure)
1173  interpolationDone = True
1174 
1175  if self.config.brighterFatterLevel == 'DETECTOR':
1176  kernelElement = bfKernel
1177  else:
1178  # TODO: DM-15631 for implementing this
1179  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
1180  self.log.info("Applying brighter fatter correction.")
1181  isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1182  self.config.brighterFatterMaxIter,
1183  self.config.brighterFatterThreshold,
1184  self.config.brighterFatterApplyGain,
1185  )
1186  self.debugView(ccdExposure, "doBrighterFatter")
1187 
1188  if self.config.doDark:
1189  self.log.info("Applying dark correction.")
1190  self.darkCorrection(ccdExposure, dark)
1191  self.debugView(ccdExposure, "doDark")
1192 
1193  if self.config.doFringe and not self.config.fringeAfterFlat:
1194  self.log.info("Applying fringe correction before flat.")
1195  self.fringe.run(ccdExposure, **fringes.getDict())
1196  self.debugView(ccdExposure, "doFringe")
1197 
1198  if self.config.doStrayLight:
1199  self.log.info("Applying stray light correction.")
1200  self.strayLight.run(ccdExposure)
1201  self.debugView(ccdExposure, "doStrayLight")
1202 
1203  if self.config.doFlat:
1204  self.log.info("Applying flat correction.")
1205  self.flatCorrection(ccdExposure, flat)
1206  self.debugView(ccdExposure, "doFlat")
1207 
1208  if self.config.doApplyGains:
1209  self.log.info("Applying gain correction instead of flat.")
1210  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1211 
1212  if self.config.doDefect and not interpolationDone:
1213  self.log.info("Masking and interpolating defects.")
1214  self.maskAndInterpDefect(ccdExposure, defects)
1215 
1216  if self.config.doSaturationInterpolation and not interpolationDone:
1217  self.log.info("Interpolating saturated pixels.")
1218  self.saturationInterpolation(ccdExposure)
1219 
1220  if self.config.doNanInterpAfterFlat or not interpolationDone:
1221  self.log.info("Masking and interpolating NAN value pixels.")
1222  self.maskAndInterpNan(ccdExposure)
1223 
1224  if self.config.doFringe and self.config.fringeAfterFlat:
1225  self.log.info("Applying fringe correction after flat.")
1226  self.fringe.run(ccdExposure, **fringes.getDict())
1227 
1228  if self.config.doSetBadRegions:
1229  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1230  if badPixelCount > 0:
1231  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1232 
1233  flattenedThumb = None
1234  if self.config.qa.doThumbnailFlattened:
1235  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1236 
1237  if self.config.doCameraSpecificMasking:
1238  self.log.info("Masking regions for camera specific reasons.")
1239  self.masking.run(ccdExposure)
1240 
1241  self.roughZeroPoint(ccdExposure)
1242 
1243  if self.config.doVignette:
1244  self.log.info("Constructing Vignette polygon.")
1245  self.vignettePolygon = self.vignette.run(ccdExposure)
1246 
1247  if self.config.vignette.doWriteVignettePolygon:
1248  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1249 
1250  if self.config.doAttachTransmissionCurve:
1251  self.log.info("Adding transmission curves.")
1252  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1253  filterTransmission=filterTransmission,
1254  sensorTransmission=sensorTransmission,
1255  atmosphereTransmission=atmosphereTransmission)
1256 
1257  if self.config.doAddDistortionModel:
1258  self.log.info("Adding a distortion model to the WCS.")
1259  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1260 
1261  if self.config.doMeasureBackground:
1262  self.log.info("Measuring background level:")
1263  self.measureBackground(ccdExposure, self.config.qa)
1264 
1265  if self.config.qa is not None and self.config.qa.saveStats is True:
1266  for amp in ccd:
1267  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1268  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1269  afwMath.MEDIAN | afwMath.STDEVCLIP)
1270  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1271  qaStats.getValue(afwMath.MEDIAN))
1272  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1273  qaStats.getValue(afwMath.STDEVCLIP))
1274  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1275  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1276  qaStats.getValue(afwMath.STDEVCLIP)))
1277 
1278  self.debugView(ccdExposure, "postISRCCD")
1279 
1280  return pipeBase.Struct(
1281  exposure=ccdExposure,
1282  ossThumb=ossThumb,
1283  flattenedThumb=flattenedThumb,
1284 
1285  outputExposure=ccdExposure,
1286  outputOssThumbnail=ossThumb,
1287  outputFlattenedThumbnail=flattenedThumb,
1288  )
1289 
1290  @pipeBase.timeMethod
1291  def runDataRef(self, sensorRef):
1292  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1293 
1294  This method contains the `CmdLineTask` interface to the ISR
1295  processing. All IO is handled here, freeing the `run()` method
1296  to manage only pixel-level calculations. The steps performed
1297  are:
1298  - Read in necessary detrending/isr/calibration data.
1299  - Process raw exposure in `run()`.
1300  - Persist the ISR-corrected exposure as "postISRCCD" if
1301  config.doWrite=True.
1302 
1303  Parameters
1304  ----------
1305  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1306  DataRef of the detector data to be processed
1307 
1308  Returns
1309  -------
1310  result : `lsst.pipe.base.Struct`
1311  Result struct with component:
1312  - ``exposure`` : `afw.image.Exposure`
1313  The fully ISR corrected exposure.
1314 
1315  Raises
1316  ------
1317  RuntimeError
1318  Raised if a configuration option is set to True, but the
1319  required calibration data does not exist.
1320 
1321  """
1322  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1323 
1324  ccdExposure = sensorRef.get(self.config.datasetType)
1325 
1326  camera = sensorRef.get("camera")
1327  if camera is None and self.config.doAddDistortionModel:
1328  raise RuntimeError("config.doAddDistortionModel is True "
1329  "but could not get a camera from the butler")
1330  isrData = self.readIsrData(sensorRef, ccdExposure)
1331 
1332  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1333 
1334  if self.config.doWrite:
1335  sensorRef.put(result.exposure, "postISRCCD")
1336  if result.ossThumb is not None:
1337  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1338  if result.flattenedThumb is not None:
1339  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1340 
1341  return result
1342 
1343  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1344  """!Retrieve a calibration dataset for removing instrument signature.
1345 
1346  Parameters
1347  ----------
1348 
1349  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1350  DataRef of the detector data to find calibration datasets
1351  for.
1352  datasetType : `str`
1353  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1354  immediate : `Bool`
1355  If True, disable butler proxies to enable error handling
1356  within this routine.
1357 
1358  Returns
1359  -------
1360  exposure : `lsst.afw.image.Exposure`
1361  Requested calibration frame.
1362 
1363  Raises
1364  ------
1365  RuntimeError
1366  Raised if no matching calibration frame can be found.
1367  """
1368  try:
1369  exp = dataRef.get(datasetType, immediate=immediate)
1370  except Exception as exc1:
1371  if not self.config.fallbackFilterName:
1372  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1373  try:
1374  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1375  except Exception as exc2:
1376  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1377  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1378  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1379 
1380  if self.config.doAssembleIsrExposures:
1381  exp = self.assembleCcd.assembleCcd(exp)
1382  return exp
1383 
1384  def ensureExposure(self, inputExp, camera, detectorNum):
1385  """Ensure that the data returned by Butler is a fully constructed exposure.
1386 
1387  ISR requires exposure-level image data for historical reasons, so if we did
1388  not recieve that from Butler, construct it from what we have, modifying the
1389  input in place.
1390 
1391  Parameters
1392  ----------
1393  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1394  `lsst.afw.image.ImageF`
1395  The input data structure obtained from Butler.
1396  camera : `lsst.afw.cameraGeom.camera`
1397  The camera associated with the image. Used to find the appropriate
1398  detector.
1399  detectorNum : `int`
1400  The detector this exposure should match.
1401 
1402  Returns
1403  -------
1404  inputExp : `lsst.afw.image.Exposure`
1405  The re-constructed exposure, with appropriate detector parameters.
1406 
1407  Raises
1408  ------
1409  TypeError
1410  Raised if the input data cannot be used to construct an exposure.
1411  """
1412  if isinstance(inputExp, afwImage.DecoratedImageU):
1413  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1414  elif isinstance(inputExp, afwImage.ImageF):
1415  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1416  elif isinstance(inputExp, afwImage.MaskedImageF):
1417  inputExp = afwImage.makeExposure(inputExp)
1418  elif isinstance(inputExp, afwImage.Exposure):
1419  pass
1420  else:
1421  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1422 
1423  if inputExp.getDetector() is None:
1424  inputExp.setDetector(camera[detectorNum])
1425 
1426  return inputExp
1427 
1428  def convertIntToFloat(self, exposure):
1429  """Convert exposure image from uint16 to float.
1430 
1431  If the exposure does not need to be converted, the input is
1432  immediately returned. For exposures that are converted to use
1433  floating point pixels, the variance is set to unity and the
1434  mask to zero.
1435 
1436  Parameters
1437  ----------
1438  exposure : `lsst.afw.image.Exposure`
1439  The raw exposure to be converted.
1440 
1441  Returns
1442  -------
1443  newexposure : `lsst.afw.image.Exposure`
1444  The input ``exposure``, converted to floating point pixels.
1445 
1446  Raises
1447  ------
1448  RuntimeError
1449  Raised if the exposure type cannot be converted to float.
1450 
1451  """
1452  if isinstance(exposure, afwImage.ExposureF):
1453  # Nothing to be done
1454  return exposure
1455  if not hasattr(exposure, "convertF"):
1456  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1457 
1458  newexposure = exposure.convertF()
1459  newexposure.variance[:] = 1
1460  newexposure.mask[:] = 0x0
1461 
1462  return newexposure
1463 
1464  def maskAmplifier(self, ccdExposure, amp, defects):
1465  """Identify bad amplifiers, saturated and suspect pixels.
1466 
1467  Parameters
1468  ----------
1469  ccdExposure : `lsst.afw.image.Exposure`
1470  Input exposure to be masked.
1471  amp : `lsst.afw.table.AmpInfoCatalog`
1472  Catalog of parameters defining the amplifier on this
1473  exposure to mask.
1474  defects : `list`
1475  List of defects. Used to determine if the entire
1476  amplifier is bad.
1477 
1478  Returns
1479  -------
1480  badAmp : `Bool`
1481  If this is true, the entire amplifier area is covered by
1482  defects and unusable.
1483 
1484  """
1485  maskedImage = ccdExposure.getMaskedImage()
1486 
1487  badAmp = False
1488 
1489  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1490  # comparison with current defects definition.
1491  if defects is not None:
1492  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1493 
1494  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1495  # association with pixels in current ccdExposure).
1496  if badAmp:
1497  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1498  afwImage.PARENT)
1499  maskView = dataView.getMask()
1500  maskView |= maskView.getPlaneBitMask("BAD")
1501  del maskView
1502  return badAmp
1503 
1504  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1505  # Saturation and suspect pixels can be masked now, though.
1506  limits = dict()
1507  if self.config.doSaturation and not badAmp:
1508  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1509  if self.config.doSuspect and not badAmp:
1510  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1511 
1512  for maskName, maskThreshold in limits.items():
1513  if not math.isnan(maskThreshold):
1514  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1515  isrFunctions.makeThresholdMask(
1516  maskedImage=dataView,
1517  threshold=maskThreshold,
1518  growFootprints=0,
1519  maskName=maskName
1520  )
1521 
1522  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1523  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1524  afwImage.PARENT)
1525  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1526  self.config.suspectMaskName])
1527  if numpy.all(maskView.getArray() & maskVal > 0):
1528  badAmp = True
1529 
1530  return badAmp
1531 
1532  def overscanCorrection(self, ccdExposure, amp):
1533  """Apply overscan correction in place.
1534 
1535  This method does initial pixel rejection of the overscan
1536  region. The overscan can also be optionally segmented to
1537  allow for discontinuous overscan responses to be fit
1538  separately. The actual overscan subtraction is performed by
1539  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1540  which is called here after the amplifier is preprocessed.
1541 
1542  Parameters
1543  ----------
1544  ccdExposure : `lsst.afw.image.Exposure`
1545  Exposure to have overscan correction performed.
1546  amp : `lsst.afw.table.AmpInfoCatalog`
1547  The amplifier to consider while correcting the overscan.
1548 
1549  Returns
1550  -------
1551  overscanResults : `lsst.pipe.base.Struct`
1552  Result struct with components:
1553  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1554  Value or fit subtracted from the amplifier image data.
1555  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1556  Value or fit subtracted from the overscan image data.
1557  - ``overscanImage`` : `lsst.afw.image.Image`
1558  Image of the overscan region with the overscan
1559  correction applied. This quantity is used to estimate
1560  the amplifier read noise empirically.
1561 
1562  Raises
1563  ------
1564  RuntimeError
1565  Raised if the ``amp`` does not contain raw pixel information.
1566 
1567  See Also
1568  --------
1569  lsst.ip.isr.isrFunctions.overscanCorrection
1570  """
1571  if not amp.getHasRawInfo():
1572  raise RuntimeError("This method must be executed on an amp with raw information.")
1573 
1574  if amp.getRawHorizontalOverscanBBox().isEmpty():
1575  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1576  return None
1577 
1578  statControl = afwMath.StatisticsControl()
1579  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1580 
1581  # Determine the bounding boxes
1582  dataBBox = amp.getRawDataBBox()
1583  oscanBBox = amp.getRawHorizontalOverscanBBox()
1584  dx0 = 0
1585  dx1 = 0
1586 
1587  prescanBBox = amp.getRawPrescanBBox()
1588  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1589  dx0 += self.config.overscanNumLeadingColumnsToSkip
1590  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1591  else:
1592  dx0 += self.config.overscanNumTrailingColumnsToSkip
1593  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1594 
1595  # Determine if we need to work on subregions of the amplifier and overscan.
1596  imageBBoxes = []
1597  overscanBBoxes = []
1598 
1599  if ((self.config.overscanBiasJump and
1600  self.config.overscanBiasJumpLocation) and
1601  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1602  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1603  self.config.overscanBiasJumpDevices)):
1604  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1605  yLower = self.config.overscanBiasJumpLocation
1606  yUpper = dataBBox.getHeight() - yLower
1607  else:
1608  yUpper = self.config.overscanBiasJumpLocation
1609  yLower = dataBBox.getHeight() - yUpper
1610 
1611  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1612  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1613  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1614  afwGeom.Extent2I(dx0, 0),
1615  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1616  yLower)))
1617 
1618  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1619  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1620  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1621  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1622  yUpper)))
1623  else:
1624  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1625  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1626  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1627  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1628  oscanBBox.getHeight())))
1629 
1630  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1631  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1632  ampImage = ccdExposure.maskedImage[imageBBox]
1633  overscanImage = ccdExposure.maskedImage[overscanBBox]
1634 
1635  overscanArray = overscanImage.image.array
1636  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1637  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1638  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1639 
1640  statControl = afwMath.StatisticsControl()
1641  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1642 
1643  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1644  overscanImage=overscanImage,
1645  fitType=self.config.overscanFitType,
1646  order=self.config.overscanOrder,
1647  collapseRej=self.config.overscanNumSigmaClip,
1648  statControl=statControl,
1649  overscanIsInt=self.config.overscanIsInt
1650  )
1651 
1652  # Measure average overscan levels and record them in the metadata
1653  levelStat = afwMath.MEDIAN
1654  sigmaStat = afwMath.STDEVCLIP
1655 
1656  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1657  self.config.qa.flatness.nIter)
1658  metadata = ccdExposure.getMetadata()
1659  ampNum = amp.getName()
1660  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1661  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1662  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1663  else:
1664  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1665  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1666  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1667 
1668  return overscanResults
1669 
1670  def updateVariance(self, ampExposure, amp, overscanImage=None):
1671  """Set the variance plane using the amplifier gain and read noise
1672 
1673  The read noise is calculated from the ``overscanImage`` if the
1674  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1675  the value from the amplifier data is used.
1676 
1677  Parameters
1678  ----------
1679  ampExposure : `lsst.afw.image.Exposure`
1680  Exposure to process.
1681  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1682  Amplifier detector data.
1683  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1684  Image of overscan, required only for empirical read noise.
1685 
1686  See also
1687  --------
1688  lsst.ip.isr.isrFunctions.updateVariance
1689  """
1690  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1691  gain = amp.getGain()
1692 
1693  if math.isnan(gain):
1694  gain = 1.0
1695  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1696  elif gain <= 0:
1697  patchedGain = 1.0
1698  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1699  (amp.getName(), gain, patchedGain))
1700  gain = patchedGain
1701 
1702  if self.config.doEmpiricalReadNoise and overscanImage is None:
1703  self.log.info("Overscan is none for EmpiricalReadNoise")
1704 
1705  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1706  stats = afwMath.StatisticsControl()
1707  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1708  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1709  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1710  else:
1711  readNoise = amp.getReadNoise()
1712 
1713  isrFunctions.updateVariance(
1714  maskedImage=ampExposure.getMaskedImage(),
1715  gain=gain,
1716  readNoise=readNoise,
1717  )
1718 
1719  def darkCorrection(self, exposure, darkExposure, invert=False):
1720  """!Apply dark correction in place.
1721 
1722  Parameters
1723  ----------
1724  exposure : `lsst.afw.image.Exposure`
1725  Exposure to process.
1726  darkExposure : `lsst.afw.image.Exposure`
1727  Dark exposure of the same size as ``exposure``.
1728  invert : `Bool`, optional
1729  If True, re-add the dark to an already corrected image.
1730 
1731  Raises
1732  ------
1733  RuntimeError
1734  Raised if either ``exposure`` or ``darkExposure`` do not
1735  have their dark time defined.
1736 
1737  See Also
1738  --------
1739  lsst.ip.isr.isrFunctions.darkCorrection
1740  """
1741  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1742  if math.isnan(expScale):
1743  raise RuntimeError("Exposure darktime is NAN")
1744  if darkExposure.getInfo().getVisitInfo() is not None:
1745  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1746  else:
1747  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1748  # so getDarkTime() does not exist.
1749  darkScale = 1.0
1750 
1751  if math.isnan(darkScale):
1752  raise RuntimeError("Dark calib darktime is NAN")
1753  isrFunctions.darkCorrection(
1754  maskedImage=exposure.getMaskedImage(),
1755  darkMaskedImage=darkExposure.getMaskedImage(),
1756  expScale=expScale,
1757  darkScale=darkScale,
1758  invert=invert,
1759  trimToFit=self.config.doTrimToMatchCalib
1760  )
1761 
1762  def doLinearize(self, detector):
1763  """!Check if linearization is needed for the detector cameraGeom.
1764 
1765  Checks config.doLinearize and the linearity type of the first
1766  amplifier.
1767 
1768  Parameters
1769  ----------
1770  detector : `lsst.afw.cameraGeom.Detector`
1771  Detector to get linearity type from.
1772 
1773  Returns
1774  -------
1775  doLinearize : `Bool`
1776  If True, linearization should be performed.
1777  """
1778  return self.config.doLinearize and \
1779  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1780 
1781  def flatCorrection(self, exposure, flatExposure, invert=False):
1782  """!Apply flat correction in place.
1783 
1784  Parameters
1785  ----------
1786  exposure : `lsst.afw.image.Exposure`
1787  Exposure to process.
1788  flatExposure : `lsst.afw.image.Exposure`
1789  Flat exposure of the same size as ``exposure``.
1790  invert : `Bool`, optional
1791  If True, unflatten an already flattened image.
1792 
1793  See Also
1794  --------
1795  lsst.ip.isr.isrFunctions.flatCorrection
1796  """
1797  isrFunctions.flatCorrection(
1798  maskedImage=exposure.getMaskedImage(),
1799  flatMaskedImage=flatExposure.getMaskedImage(),
1800  scalingType=self.config.flatScalingType,
1801  userScale=self.config.flatUserScale,
1802  invert=invert,
1803  trimToFit=self.config.doTrimToMatchCalib
1804  )
1805 
1806  def saturationDetection(self, exposure, amp):
1807  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1808 
1809  Parameters
1810  ----------
1811  exposure : `lsst.afw.image.Exposure`
1812  Exposure to process. Only the amplifier DataSec is processed.
1813  amp : `lsst.afw.table.AmpInfoCatalog`
1814  Amplifier detector data.
1815 
1816  See Also
1817  --------
1818  lsst.ip.isr.isrFunctions.makeThresholdMask
1819  """
1820  if not math.isnan(amp.getSaturation()):
1821  maskedImage = exposure.getMaskedImage()
1822  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1823  isrFunctions.makeThresholdMask(
1824  maskedImage=dataView,
1825  threshold=amp.getSaturation(),
1826  growFootprints=0,
1827  maskName=self.config.saturatedMaskName,
1828  )
1829 
1830  def saturationInterpolation(self, ccdExposure):
1831  """!Interpolate over saturated pixels, in place.
1832 
1833  This method should be called after `saturationDetection`, to
1834  ensure that the saturated pixels have been identified in the
1835  SAT mask. It should also be called after `assembleCcd`, since
1836  saturated regions may cross amplifier boundaries.
1837 
1838  Parameters
1839  ----------
1840  exposure : `lsst.afw.image.Exposure`
1841  Exposure to process.
1842 
1843  See Also
1844  --------
1845  lsst.ip.isr.isrTask.saturationDetection
1846  lsst.ip.isr.isrFunctions.interpolateFromMask
1847  """
1848  isrFunctions.interpolateFromMask(
1849  maskedImage=ccdExposure.getMaskedImage(),
1850  fwhm=self.config.fwhm,
1851  growFootprints=self.config.growSaturationFootprintSize,
1852  maskName=self.config.saturatedMaskName,
1853  )
1854 
1855  def suspectDetection(self, exposure, amp):
1856  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1857 
1858  Parameters
1859  ----------
1860  exposure : `lsst.afw.image.Exposure`
1861  Exposure to process. Only the amplifier DataSec is processed.
1862  amp : `lsst.afw.table.AmpInfoCatalog`
1863  Amplifier detector data.
1864 
1865  See Also
1866  --------
1867  lsst.ip.isr.isrFunctions.makeThresholdMask
1868 
1869  Notes
1870  -----
1871  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1872  This is intended to indicate pixels that may be affected by unknown systematics;
1873  for example if non-linearity corrections above a certain level are unstable
1874  then that would be a useful value for suspectLevel. A value of `nan` indicates
1875  that no such level exists and no pixels are to be masked as suspicious.
1876  """
1877  suspectLevel = amp.getSuspectLevel()
1878  if math.isnan(suspectLevel):
1879  return
1880 
1881  maskedImage = exposure.getMaskedImage()
1882  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1883  isrFunctions.makeThresholdMask(
1884  maskedImage=dataView,
1885  threshold=suspectLevel,
1886  growFootprints=0,
1887  maskName=self.config.suspectMaskName,
1888  )
1889 
1890  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1891  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1892 
1893  Parameters
1894  ----------
1895  ccdExposure : `lsst.afw.image.Exposure`
1896  Exposure to process.
1897  defectBaseList : `List`
1898  List of defects to mask and interpolate.
1899 
1900  Notes
1901  -----
1902  Call this after CCD assembly, since defects may cross amplifier boundaries.
1903  """
1904  maskedImage = ccdExposure.getMaskedImage()
1905  defectList = []
1906  for d in defectBaseList:
1907  bbox = d.getBBox()
1908  nd = measAlg.Defect(bbox)
1909  defectList.append(nd)
1910  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1911  isrFunctions.interpolateDefectList(
1912  maskedImage=maskedImage,
1913  defectList=defectList,
1914  fwhm=self.config.fwhm,
1915  )
1916 
1917  if self.config.numEdgeSuspect > 0:
1918  goodBBox = maskedImage.getBBox()
1919  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1920  goodBBox.grow(-self.config.numEdgeSuspect)
1921  # Mask pixels outside goodBBox as SUSPECT
1922  SourceDetectionTask.setEdgeBits(
1923  maskedImage,
1924  goodBBox,
1925  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1926  )
1927 
1928  def maskAndInterpNan(self, exposure):
1929  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1930 
1931  Parameters
1932  ----------
1933  exposure : `lsst.afw.image.Exposure`
1934  Exposure to process.
1935 
1936  Notes
1937  -----
1938  We mask and interpolate over all NaNs, including those
1939  that are masked with other bits (because those may or may
1940  not be interpolated over later, and we want to remove all
1941  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1942  is used to preserve the historical name.
1943  """
1944  maskedImage = exposure.getMaskedImage()
1945 
1946  # Find and mask NaNs
1947  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1948  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1949  numNans = maskNans(maskedImage, maskVal)
1950  self.metadata.set("NUMNANS", numNans)
1951 
1952  # Interpolate over these previously-unmasked NaNs
1953  if numNans > 0:
1954  self.log.warn("There were %i unmasked NaNs", numNans)
1955  nanDefectList = isrFunctions.getDefectListFromMask(
1956  maskedImage=maskedImage,
1957  maskName='UNMASKEDNAN',
1958  )
1959  isrFunctions.interpolateDefectList(
1960  maskedImage=exposure.getMaskedImage(),
1961  defectList=nanDefectList,
1962  fwhm=self.config.fwhm,
1963  )
1964 
1965  def measureBackground(self, exposure, IsrQaConfig=None):
1966  """Measure the image background in subgrids, for quality control purposes.
1967 
1968  Parameters
1969  ----------
1970  exposure : `lsst.afw.image.Exposure`
1971  Exposure to process.
1972  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
1973  Configuration object containing parameters on which background
1974  statistics and subgrids to use.
1975  """
1976  if IsrQaConfig is not None:
1977  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1978  IsrQaConfig.flatness.nIter)
1979  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
1980  statsControl.setAndMask(maskVal)
1981  maskedImage = exposure.getMaskedImage()
1982  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1983  skyLevel = stats.getValue(afwMath.MEDIAN)
1984  skySigma = stats.getValue(afwMath.STDEVCLIP)
1985  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1986  metadata = exposure.getMetadata()
1987  metadata.set('SKYLEVEL', skyLevel)
1988  metadata.set('SKYSIGMA', skySigma)
1989 
1990  # calcluating flatlevel over the subgrids
1991  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
1992  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1993  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
1994  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
1995  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
1996  skyLevels = numpy.zeros((nX, nY))
1997 
1998  for j in range(nY):
1999  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2000  for i in range(nX):
2001  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2002 
2003  xLLC = xc - meshXHalf
2004  yLLC = yc - meshYHalf
2005  xURC = xc + meshXHalf - 1
2006  yURC = yc + meshYHalf - 1
2007 
2008  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2009  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2010 
2011  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2012 
2013  good = numpy.where(numpy.isfinite(skyLevels))
2014  skyMedian = numpy.median(skyLevels[good])
2015  flatness = (skyLevels[good] - skyMedian) / skyMedian
2016  flatness_rms = numpy.std(flatness)
2017  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2018 
2019  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2020  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2021  (nX, nY, flatness_pp, flatness_rms))
2022 
2023  metadata.set('FLATNESS_PP', float(flatness_pp))
2024  metadata.set('FLATNESS_RMS', float(flatness_rms))
2025  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2026  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2027  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2028 
2029  def roughZeroPoint(self, exposure):
2030  """Set an approximate magnitude zero point for the exposure.
2031 
2032  Parameters
2033  ----------
2034  exposure : `lsst.afw.image.Exposure`
2035  Exposure to process.
2036  """
2037  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2038  if filterName in self.config.fluxMag0T1:
2039  fluxMag0 = self.config.fluxMag0T1[filterName]
2040  else:
2041  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2042  fluxMag0 = self.config.defaultFluxMag0T1
2043 
2044  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2045  if not expTime > 0: # handle NaN as well as <= 0
2046  self.log.warn("Non-positive exposure time; skipping rough zero point")
2047  return
2048 
2049  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2050  exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2051 
2052  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2053  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2054 
2055  Parameters
2056  ----------
2057  ccdExposure : `lsst.afw.image.Exposure`
2058  Exposure to process.
2059  fpPolygon : `lsst.afw.geom.Polygon`
2060  Polygon in focal plane coordinates.
2061  """
2062  # Get ccd corners in focal plane coordinates
2063  ccd = ccdExposure.getDetector()
2064  fpCorners = ccd.getCorners(FOCAL_PLANE)
2065  ccdPolygon = Polygon(fpCorners)
2066 
2067  # Get intersection of ccd corners with fpPolygon
2068  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2069 
2070  # Transform back to pixel positions and build new polygon
2071  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2072  validPolygon = Polygon(ccdPoints)
2073  ccdExposure.getInfo().setValidPolygon(validPolygon)
2074 
2075  @contextmanager
2076  def flatContext(self, exp, flat, dark=None):
2077  """Context manager that applies and removes flats and darks,
2078  if the task is configured to apply them.
2079 
2080  Parameters
2081  ----------
2082  exp : `lsst.afw.image.Exposure`
2083  Exposure to process.
2084  flat : `lsst.afw.image.Exposure`
2085  Flat exposure the same size as ``exp``.
2086  dark : `lsst.afw.image.Exposure`, optional
2087  Dark exposure the same size as ``exp``.
2088 
2089  Yields
2090  ------
2091  exp : `lsst.afw.image.Exposure`
2092  The flat and dark corrected exposure.
2093  """
2094  if self.config.doDark and dark is not None:
2095  self.darkCorrection(exp, dark)
2096  if self.config.doFlat:
2097  self.flatCorrection(exp, flat)
2098  try:
2099  yield exp
2100  finally:
2101  if self.config.doFlat:
2102  self.flatCorrection(exp, flat, invert=True)
2103  if self.config.doDark and dark is not None:
2104  self.darkCorrection(exp, dark, invert=True)
2105 
2106  def debugView(self, exposure, stepname):
2107  """Utility function to examine ISR exposure at different stages.
2108 
2109  Parameters
2110  ----------
2111  exposure : `lsst.afw.image.Exposure`
2112  Exposure to view.
2113  stepname : `str`
2114  State of processing to view.
2115  """
2116  frame = getDebugFrame(self._display, stepname)
2117  if frame:
2118  display = getDisplay(frame)
2119  display.scale('asinh', 'zscale')
2120  display.mtv(exposure)
2121 
2122 
2123 class FakeAmp(object):
2124  """A Detector-like object that supports returning gain and saturation level
2125 
2126  This is used when the input exposure does not have a detector.
2127 
2128  Parameters
2129  ----------
2130  exposure : `lsst.afw.image.Exposure`
2131  Exposure to generate a fake amplifier for.
2132  config : `lsst.ip.isr.isrTaskConfig`
2133  Configuration to apply to the fake amplifier.
2134  """
2135 
2136  def __init__(self, exposure, config):
2137  self._bbox = exposure.getBBox(afwImage.LOCAL)
2138  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
2139  self._gain = config.gain
2140  self._readNoise = config.readNoise
2141  self._saturation = config.saturation
2142 
2143  def getBBox(self):
2144  return self._bbox
2145 
2146  def getRawBBox(self):
2147  return self._bbox
2148 
2149  def getHasRawInfo(self):
2150  return True # but see getRawHorizontalOverscanBBox()
2151 
2153  return self._RawHorizontalOverscanBBox
2154 
2155  def getGain(self):
2156  return self._gain
2157 
2158  def getReadNoise(self):
2159  return self._readNoise
2160 
2161  def getSaturation(self):
2162  return self._saturation
2163 
2164  def getSuspectLevel(self):
2165  return float("NaN")
2166 
2167 
2168 class RunIsrConfig(pexConfig.Config):
2169  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2170 
2171 
2172 class RunIsrTask(pipeBase.CmdLineTask):
2173  """Task to wrap the default IsrTask to allow it to be retargeted.
2174 
2175  The standard IsrTask can be called directly from a command line
2176  program, but doing so removes the ability of the task to be
2177  retargeted. As most cameras override some set of the IsrTask
2178  methods, this would remove those data-specific methods in the
2179  output post-ISR images. This wrapping class fixes the issue,
2180  allowing identical post-ISR images to be generated by both the
2181  processCcd and isrTask code.
2182  """
2183  ConfigClass = RunIsrConfig
2184  _DefaultName = "runIsr"
2185 
2186  def __init__(self, *args, **kwargs):
2187  super().__init__(*args, **kwargs)
2188  self.makeSubtask("isr")
2189 
2190  def runDataRef(self, dataRef):
2191  """
2192  Parameters
2193  ----------
2194  dataRef : `lsst.daf.persistence.ButlerDataRef`
2195  data reference of the detector data to be processed
2196 
2197  Returns
2198  -------
2199  result : `pipeBase.Struct`
2200  Result struct with component:
2201 
2202  - exposure : `lsst.afw.image.Exposure`
2203  Post-ISR processed exposure.
2204  """
2205  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:710
def runDataRef(self, sensorRef)
Definition: isrTask.py:1291
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:1965
def debugView(self, exposure, stepname)
Definition: isrTask.py:2106
def __init__(self, kwargs)
Definition: isrTask.py:700
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1384
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:805
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:759
def runDataRef(self, dataRef)
Definition: isrTask.py:2190
def __init__(self, args, kwargs)
Definition: isrTask.py:2186
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1928
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1830
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2029
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2152
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:747
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1532
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1428
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1781
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:802
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1343
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1719
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1762
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:909
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2052
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1464
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2076
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1670
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1855
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1890
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1806
def __init__(self, exposure, config)
Definition: isrTask.py:2136