lsst.ip.isr  17.0.1-9-g77829d8+6
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defect
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 __all__ = ["IsrTask", "RunIsrTask"]
57 
58 
59 class IsrTaskConfig(pexConfig.Config):
60  """Configuration parameters for IsrTask.
61 
62  Items are grouped in the order in which they are executed by the task.
63  """
64  # General ISR configuration
65 
66  # gen3 options
67  isrName = pexConfig.Field(
68  dtype=str,
69  doc="Name of ISR",
70  default="ISR",
71  )
72 
73  # input datasets
74  ccdExposure = pipeBase.InputDatasetField(
75  doc="Input exposure to process",
76  name="raw",
77  scalar=True,
78  storageClass="ExposureU",
79  dimensions=["Instrument", "Exposure", "Detector"],
80  )
81  camera = pipeBase.InputDatasetField(
82  doc="Input camera to construct complete exposures.",
83  name="camera",
84  scalar=True,
85  storageClass="TablePersistableCamera",
86  dimensions=["Instrument", "CalibrationLabel"],
87  )
88  bias = pipeBase.InputDatasetField(
89  doc="Input bias calibration.",
90  name="bias",
91  scalar=True,
92  storageClass="ImageF",
93  dimensions=["Instrument", "CalibrationLabel", "Detector"],
94  )
95  dark = pipeBase.InputDatasetField(
96  doc="Input dark calibration.",
97  name="dark",
98  scalar=True,
99  storageClass="ImageF",
100  dimensions=["Instrument", "CalibrationLabel", "Detector"],
101  )
102  flat = pipeBase.InputDatasetField(
103  doc="Input flat calibration.",
104  name="flat",
105  scalar=True,
106  storageClass="MaskedImageF",
107  dimensions=["Instrument", "PhysicalFilter", "CalibrationLabel", "Detector"],
108  )
109  bfKernel = pipeBase.InputDatasetField(
110  doc="Input brighter-fatter kernel.",
111  name="bfKernel",
112  scalar=True,
113  storageClass="NumpyArray",
114  dimensions=["Instrument", "CalibrationLabel"],
115  )
116  defects = pipeBase.InputDatasetField(
117  doc="Input defect tables.",
118  name="defects",
119  scalar=True,
120  storageClass="Catalog",
121  dimensions=["Instrument", "CalibrationLabel", "Detector"],
122  )
123  opticsTransmission = pipeBase.InputDatasetField(
124  doc="Transmission curve due to the optics.",
125  name="transmission_optics",
126  scalar=True,
127  storageClass="TablePersistableTransmissionCurve",
128  dimensions=["Instrument", "CalibrationLabel"],
129  )
130  filterTransmission = pipeBase.InputDatasetField(
131  doc="Transmission curve due to the filter.",
132  name="transmission_filter",
133  scalar=True,
134  storageClass="TablePersistableTransmissionCurve",
135  dimensions=["Instrument", "PhysicalFilter", "CalibrationLabel"],
136  )
137  sensorTransmission = pipeBase.InputDatasetField(
138  doc="Transmission curve due to the sensor.",
139  name="transmission_sensor",
140  scalar=True,
141  storageClass="TablePersistableTransmissionCurve",
142  dimensions=["Instrument", "CalibrationLabel", "Detector"],
143  )
144  atmosphereTransmission = pipeBase.InputDatasetField(
145  doc="Transmission curve due to the atmosphere.",
146  name="transmission_atmosphere",
147  scalar=True,
148  storageClass="TablePersistableTransmissionCurve",
149  dimensions=["Instrument"],
150  )
151 
152  # output datasets
153  outputExposure = pipeBase.OutputDatasetField(
154  doc="Output ISR processed exposure.",
155  name="postISRCCD",
156  scalar=True,
157  storageClass="ExposureF",
158  dimensions=["Instrument", "Visit", "Detector"],
159  )
160  outputOssThumbnail = pipeBase.OutputDatasetField(
161  doc="Output Overscan-subtracted thumbnail image.",
162  name="OssThumb",
163  scalar=True,
164  storageClass="Thumbnail",
165  dimensions=["Instrument", "Visit", "Detector"],
166  )
167  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
168  doc="Output flat-corrected thumbnail image.",
169  name="FlattenedThumb",
170  scalar=True,
171  storageClass="TextStorage",
172  dimensions=["Instrument", "Visit", "Detector"],
173  )
174 
175  quantum = pipeBase.QuantumConfig(
176  dimensions=["Visit", "Detector", "Instrument"],
177  )
178 
179 
180  datasetType = pexConfig.Field(
181  dtype=str,
182  doc="Dataset type for input data; users will typically leave this alone, "
183  "but camera-specific ISR tasks will override it",
184  default="raw",
185  )
186 
187  fallbackFilterName = pexConfig.Field(
188  dtype=str,
189  doc="Fallback default filter name for calibrations.",
190  optional=True
191  )
192  expectWcs = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
196  )
197  fwhm = pexConfig.Field(
198  dtype=float,
199  doc="FWHM of PSF in arcseconds.",
200  default=1.0,
201  )
202  qa = pexConfig.ConfigField(
203  dtype=isrQa.IsrQaConfig,
204  doc="QA related configuration options.",
205  )
206 
207  # Image conversion configuration
208  doConvertIntToFloat = pexConfig.Field(
209  dtype=bool,
210  doc="Convert integer raw images to floating point values?",
211  default=True,
212  )
213 
214  # Saturated pixel handling.
215  doSaturation = pexConfig.Field(
216  dtype=bool,
217  doc="Mask saturated pixels? NB: this is totally independent of the"
218  " interpolation option - this is ONLY setting the bits in the mask."
219  " To have them interpolated make sure doSaturationInterpolation=True",
220  default=True,
221  )
222  saturatedMaskName = pexConfig.Field(
223  dtype=str,
224  doc="Name of mask plane to use in saturation detection and interpolation",
225  default="SAT",
226  )
227  saturation = pexConfig.Field(
228  dtype=float,
229  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
230  default=float("NaN"),
231  )
232  growSaturationFootprintSize = pexConfig.Field(
233  dtype=int,
234  doc="Number of pixels by which to grow the saturation footprints",
235  default=1,
236  )
237 
238  # Suspect pixel handling.
239  doSuspect = pexConfig.Field(
240  dtype=bool,
241  doc="Mask suspect pixels?",
242  default=True,
243  )
244  suspectMaskName = pexConfig.Field(
245  dtype=str,
246  doc="Name of mask plane to use for suspect pixels",
247  default="SUSPECT",
248  )
249  numEdgeSuspect = pexConfig.Field(
250  dtype=int,
251  doc="Number of edge pixels to be flagged as untrustworthy.",
252  default=0,
253  )
254 
255  # Initial masking options.
256  doSetBadRegions = pexConfig.Field(
257  dtype=bool,
258  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
259  default=True,
260  )
261  badStatistic = pexConfig.ChoiceField(
262  dtype=str,
263  doc="How to estimate the average value for BAD regions.",
264  default='MEANCLIP',
265  allowed={
266  "MEANCLIP": "Correct using the (clipped) mean of good data",
267  "MEDIAN": "Correct using the median of the good data",
268  },
269  )
270 
271  # Overscan subtraction configuration.
272  doOverscan = pexConfig.Field(
273  dtype=bool,
274  doc="Do overscan subtraction?",
275  default=True,
276  )
277  overscanFitType = pexConfig.ChoiceField(
278  dtype=str,
279  doc="The method for fitting the overscan bias level.",
280  default='MEDIAN',
281  allowed={
282  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
283  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
284  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
285  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
286  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
287  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
288  "MEAN": "Correct using the mean of the overscan region",
289  "MEANCLIP": "Correct using a clipped mean of the overscan region",
290  "MEDIAN": "Correct using the median of the overscan region",
291  },
292  )
293  overscanOrder = pexConfig.Field(
294  dtype=int,
295  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
296  "or number of spline knots if overscan fit type is a spline."),
297  default=1,
298  )
299  overscanNumSigmaClip = pexConfig.Field(
300  dtype=float,
301  doc="Rejection threshold (sigma) for collapsing overscan before fit",
302  default=3.0,
303  )
304  overscanIsInt = pexConfig.Field(
305  dtype=bool,
306  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
307  default=True,
308  )
309  overscanNumLeadingColumnsToSkip = pexConfig.Field(
310  dtype=int,
311  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
312  default=0,
313  )
314  overscanNumTrailingColumnsToSkip = pexConfig.Field(
315  dtype=int,
316  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
317  default=0,
318  )
319  overscanMaxDev = pexConfig.Field(
320  dtype=float,
321  doc="Maximum deviation from the median for overscan",
322  default=1000.0, check=lambda x: x > 0
323  )
324  overscanBiasJump = pexConfig.Field(
325  dtype=bool,
326  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
327  default=False,
328  )
329  overscanBiasJumpKeyword = pexConfig.Field(
330  dtype=str,
331  doc="Header keyword containing information about devices.",
332  default="NO_SUCH_KEY",
333  )
334  overscanBiasJumpDevices = pexConfig.ListField(
335  dtype=str,
336  doc="List of devices that need piecewise overscan correction.",
337  default=(),
338  )
339  overscanBiasJumpLocation = pexConfig.Field(
340  dtype=int,
341  doc="Location of bias jump along y-axis.",
342  default=0,
343  )
344 
345  # Amplifier to CCD assembly configuration
346  doAssembleCcd = pexConfig.Field(
347  dtype=bool,
348  default=True,
349  doc="Assemble amp-level exposures into a ccd-level exposure?"
350  )
351  assembleCcd = pexConfig.ConfigurableField(
352  target=AssembleCcdTask,
353  doc="CCD assembly task",
354  )
355 
356  # General calibration configuration.
357  doAssembleIsrExposures = pexConfig.Field(
358  dtype=bool,
359  default=False,
360  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
361  )
362  doTrimToMatchCalib = pexConfig.Field(
363  dtype=bool,
364  default=False,
365  doc="Trim raw data to match calibration bounding boxes?"
366  )
367 
368  # Bias subtraction.
369  doBias = pexConfig.Field(
370  dtype=bool,
371  doc="Apply bias frame correction?",
372  default=True,
373  )
374  biasDataProductName = pexConfig.Field(
375  dtype=str,
376  doc="Name of the bias data product",
377  default="bias",
378  )
379 
380  # Variance construction
381  doVariance = pexConfig.Field(
382  dtype=bool,
383  doc="Calculate variance?",
384  default=True
385  )
386  gain = pexConfig.Field(
387  dtype=float,
388  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
389  default=float("NaN"),
390  )
391  readNoise = pexConfig.Field(
392  dtype=float,
393  doc="The read noise to use if no Detector is present in the Exposure",
394  default=0.0,
395  )
396  doEmpiricalReadNoise = pexConfig.Field(
397  dtype=bool,
398  default=False,
399  doc="Calculate empirical read noise instead of value from AmpInfo data?"
400  )
401 
402  # Linearization.
403  doLinearize = pexConfig.Field(
404  dtype=bool,
405  doc="Correct for nonlinearity of the detector's response?",
406  default=True,
407  )
408 
409  # Crosstalk.
410  doCrosstalk = pexConfig.Field(
411  dtype=bool,
412  doc="Apply intra-CCD crosstalk correction?",
413  default=False,
414  )
415  doCrosstalkBeforeAssemble = pexConfig.Field(
416  dtype=bool,
417  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
418  default=True,
419  )
420  crosstalk = pexConfig.ConfigurableField(
421  target=CrosstalkTask,
422  doc="Intra-CCD crosstalk correction",
423  )
424 
425  # Masking option prior to brighter-fatter.
426  doWidenSaturationTrails = pexConfig.Field(
427  dtype=bool,
428  doc="Widen bleed trails based on their width?",
429  default=True
430  )
431 
432  # Brighter-Fatter correction.
433  doBrighterFatter = pexConfig.Field(
434  dtype=bool,
435  default=False,
436  doc="Apply the brighter fatter correction"
437  )
438  brighterFatterLevel = pexConfig.ChoiceField(
439  dtype=str,
440  default="DETECTOR",
441  doc="The level at which to correct for brighter-fatter.",
442  allowed={
443  "AMP": "Every amplifier treated separately.",
444  "DETECTOR": "One kernel per detector",
445  }
446  )
447  brighterFatterKernelFile = pexConfig.Field(
448  dtype=str,
449  default='',
450  doc="Kernel file used for the brighter fatter correction"
451  )
452  brighterFatterMaxIter = pexConfig.Field(
453  dtype=int,
454  default=10,
455  doc="Maximum number of iterations for the brighter fatter correction"
456  )
457  brighterFatterThreshold = pexConfig.Field(
458  dtype=float,
459  default=1000,
460  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
461  " absolute value of the difference between the current corrected image and the one"
462  " from the previous iteration summed over all the pixels."
463  )
464  brighterFatterApplyGain = pexConfig.Field(
465  dtype=bool,
466  default=True,
467  doc="Should the gain be applied when applying the brighter fatter correction?"
468  )
469 
470  # Defect and bad pixel correction options.
471  doDefect = pexConfig.Field(
472  dtype=bool,
473  doc="Apply correction for CCD defects, e.g. hot pixels?",
474  default=True,
475  )
476  doSaturationInterpolation = pexConfig.Field(
477  dtype=bool,
478  doc="Perform interpolation over pixels masked as saturated?"
479  " NB: This is independent of doSaturation; if that is False this plane"
480  " will likely be blank, resulting in a no-op here.",
481  default=True,
482  )
483  numEdgeSuspect = pexConfig.Field(
484  dtype=int,
485  doc="Number of edge pixels to be flagged as untrustworthy.",
486  default=0,
487  )
488 
489  # Dark subtraction.
490  doDark = pexConfig.Field(
491  dtype=bool,
492  doc="Apply dark frame correction?",
493  default=True,
494  )
495  darkDataProductName = pexConfig.Field(
496  dtype=str,
497  doc="Name of the dark data product",
498  default="dark",
499  )
500 
501  # Camera-specific stray light removal.
502  doStrayLight = pexConfig.Field(
503  dtype=bool,
504  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
505  default=False,
506  )
507  strayLight = pexConfig.ConfigurableField(
508  target=StrayLightTask,
509  doc="y-band stray light correction"
510  )
511 
512  # Flat correction.
513  doFlat = pexConfig.Field(
514  dtype=bool,
515  doc="Apply flat field correction?",
516  default=True,
517  )
518  flatDataProductName = pexConfig.Field(
519  dtype=str,
520  doc="Name of the flat data product",
521  default="flat",
522  )
523  flatScalingType = pexConfig.ChoiceField(
524  dtype=str,
525  doc="The method for scaling the flat on the fly.",
526  default='USER',
527  allowed={
528  "USER": "Scale by flatUserScale",
529  "MEAN": "Scale by the inverse of the mean",
530  "MEDIAN": "Scale by the inverse of the median",
531  },
532  )
533  flatUserScale = pexConfig.Field(
534  dtype=float,
535  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
536  default=1.0,
537  )
538  doTweakFlat = pexConfig.Field(
539  dtype=bool,
540  doc="Tweak flats to match observed amplifier ratios?",
541  default=False
542  )
543 
544  # Amplifier normalization based on gains instead of using flats configuration.
545  doApplyGains = pexConfig.Field(
546  dtype=bool,
547  doc="Correct the amplifiers for their gains instead of applying flat correction",
548  default=False,
549  )
550  normalizeGains = pexConfig.Field(
551  dtype=bool,
552  doc="Normalize all the amplifiers in each CCD to have the same median value.",
553  default=False,
554  )
555 
556  # Fringe correction.
557  doFringe = pexConfig.Field(
558  dtype=bool,
559  doc="Apply fringe correction?",
560  default=True,
561  )
562  fringe = pexConfig.ConfigurableField(
563  target=FringeTask,
564  doc="Fringe subtraction task",
565  )
566  fringeAfterFlat = pexConfig.Field(
567  dtype=bool,
568  doc="Do fringe subtraction after flat-fielding?",
569  default=True,
570  )
571 
572  # NAN pixel interpolation option.
573  doNanInterpAfterFlat = pexConfig.Field(
574  dtype=bool,
575  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
576  "also have to interpolate them before flat-fielding."),
577  default=False,
578  )
579 
580  # Distortion model application.
581  doAddDistortionModel = pexConfig.Field(
582  dtype=bool,
583  doc="Apply a distortion model based on camera geometry to the WCS?",
584  default=True,
585  )
586 
587  # Initial CCD-level background statistics options.
588  doMeasureBackground = pexConfig.Field(
589  dtype=bool,
590  doc="Measure the background level on the reduced image?",
591  default=False,
592  )
593 
594  # Camera-specific masking configuration.
595  doCameraSpecificMasking = pexConfig.Field(
596  dtype=bool,
597  doc="Mask camera-specific bad regions?",
598  default=False,
599  )
600  masking = pexConfig.ConfigurableField(
601  target=MaskingTask,
602  doc="Masking task."
603  )
604 
605  # Default photometric calibration options.
606  fluxMag0T1 = pexConfig.DictField(
607  keytype=str,
608  itemtype=float,
609  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
610  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
611  ))
612  )
613  defaultFluxMag0T1 = pexConfig.Field(
614  dtype=float,
615  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
616  default=pow(10.0, 0.4*28.0)
617  )
618 
619  # Vignette correction configuration.
620  doVignette = pexConfig.Field(
621  dtype=bool,
622  doc="Apply vignetting parameters?",
623  default=False,
624  )
625  vignette = pexConfig.ConfigurableField(
626  target=VignetteTask,
627  doc="Vignetting task.",
628  )
629 
630  # Transmission curve configuration.
631  doAttachTransmissionCurve = pexConfig.Field(
632  dtype=bool,
633  default=False,
634  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
635  )
636  doUseOpticsTransmission = pexConfig.Field(
637  dtype=bool,
638  default=True,
639  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
640  )
641  doUseFilterTransmission = pexConfig.Field(
642  dtype=bool,
643  default=True,
644  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
645  )
646  doUseSensorTransmission = pexConfig.Field(
647  dtype=bool,
648  default=True,
649  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
650  )
651  doUseAtmosphereTransmission = pexConfig.Field(
652  dtype=bool,
653  default=True,
654  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
655  )
656 
657  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
658  doWrite = pexConfig.Field(
659  dtype=bool,
660  doc="Persist postISRCCD?",
661  default=True,
662  )
663 
664  def validate(self):
665  super().validate()
666  if self.doFlat and self.doApplyGains:
667  raise ValueError("You may not specify both doFlat and doApplyGains")
668 
669 
670 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
671  r"""Apply common instrument signature correction algorithms to a raw frame.
672 
673  The process for correcting imaging data is very similar from
674  camera to camera. This task provides a vanilla implementation of
675  doing these corrections, including the ability to turn certain
676  corrections off if they are not needed. The inputs to the primary
677  method, `run()`, are a raw exposure to be corrected and the
678  calibration data products. The raw input is a single chip sized
679  mosaic of all amps including overscans and other non-science
680  pixels. The method `runDataRef()` identifies and defines the
681  calibration data products, and is intended for use by a
682  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
683  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
684  subclassed for different camera, although the most camera specific
685  methods have been split into subtasks that can be redirected
686  appropriately.
687 
688  The __init__ method sets up the subtasks for ISR processing, using
689  the defaults from `lsst.ip.isr`.
690 
691  Parameters
692  ----------
693  args : `list`
694  Positional arguments passed to the Task constructor. None used at this time.
695  kwargs : `dict`, optional
696  Keyword arguments passed on to the Task constructor. None used at this time.
697  """
698  ConfigClass = IsrTaskConfig
699  _DefaultName = "isr"
700 
701  def __init__(self, **kwargs):
702  super().__init__(**kwargs)
703  self.makeSubtask("assembleCcd")
704  self.makeSubtask("crosstalk")
705  self.makeSubtask("strayLight")
706  self.makeSubtask("fringe")
707  self.makeSubtask("masking")
708  self.makeSubtask("vignette")
709 
710  @classmethod
711  def getInputDatasetTypes(cls, config):
712  inputTypeDict = super().getInputDatasetTypes(config)
713 
714  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
715  # need because the configuration tells us we will not be bothering with the
716  # correction that uses that IDT.
717  if config.doBias is not True:
718  inputTypeDict.pop("bias", None)
719  if config.doLinearize is not True:
720  inputTypeDict.pop("linearizer", None)
721  if config.doCrosstalk is not True:
722  inputTypeDict.pop("crosstalkSources", None)
723  if config.doBrighterFatter is not True:
724  inputTypeDict.pop("bfKernel", None)
725  if config.doDefect is not True:
726  inputTypeDict.pop("defects", None)
727  if config.doDark is not True:
728  inputTypeDict.pop("dark", None)
729  if config.doFlat is not True:
730  inputTypeDict.pop("flat", None)
731  if config.doAttachTransmissionCurve is not True:
732  inputTypeDict.pop("opticsTransmission", None)
733  inputTypeDict.pop("filterTransmission", None)
734  inputTypeDict.pop("sensorTransmission", None)
735  inputTypeDict.pop("atmosphereTransmission", None)
736  if config.doUseOpticsTransmission is not True:
737  inputTypeDict.pop("opticsTransmission", None)
738  if config.doUseFilterTransmission is not True:
739  inputTypeDict.pop("filterTransmission", None)
740  if config.doUseSensorTransmission is not True:
741  inputTypeDict.pop("sensorTransmission", None)
742  if config.doUseAtmosphereTransmission is not True:
743  inputTypeDict.pop("atmosphereTransmission", None)
744 
745  return inputTypeDict
746 
747  @classmethod
748  def getOutputDatasetTypes(cls, config):
749  outputTypeDict = super().getOutputDatasetTypes(config)
750 
751  if config.qa.doThumbnailOss is not True:
752  outputTypeDict.pop("outputOssThumbnail", None)
753  if config.qa.doThumbnailFlattened is not True:
754  outputTypeDict.pop("outputFlattenedThumbnail", None)
755  if config.doWrite is not True:
756  outputTypeDict.pop("outputExposure", None)
757 
758  return outputTypeDict
759 
760  @classmethod
761  def getPrerequisiteDatasetTypes(cls, config):
762  # Input calibration datasets should not constrain the QuantumGraph
763  # (it'd be confusing if not having flats just silently resulted in no
764  # data being processed). Our nomenclature for that is that these are
765  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
766  names = set(cls.getInputDatasetTypes(config))
767  names.remove("ccdExposure")
768  return names
769 
770  @classmethod
771  def getPerDatasetTypeDimensions(cls, config):
772  # Input calibration datasets of different types (i.e. flat, bias) need
773  # not have the same validity range. That makes CalibrationLabel
774  # (which maps directly to a validity range) a "per-DatasetType
775  # dimension".
776  return frozenset(["CalibrationLabel"])
777 
778  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
779  try:
780  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
781  except Exception as e:
782  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
783 
784  inputData['isGen3'] = True
785 
786  if self.config.doLinearize is True:
787  if 'linearizer' not in inputData.keys():
788  detector = inputData['camera'][inputData['detectorNum']]
789  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
790  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
791 
792  if inputData['defects'] is not None:
793  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
794  # masking expects a list of defects defined by their bounding box
795  defectList = []
796 
797  for r in inputData['defects']:
798  bbox = afwGeom.BoxI(afwGeom.PointI(r.get("x0"), r.get("y0")),
799  afwGeom.ExtentI(r.get("width"), r.get("height")))
800  defectList.append(Defect(bbox))
801 
802  inputData['defects'] = defectList
803 
804  # Broken: DM-17169
805  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
806  # fixed for non-HSC cameras in the future.
807  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
808  # if self.config.doCrosstalk else None)
809 
810  # Broken: DM-17152
811  # Fringes are not tested to be handled correctly by Gen3 butler.
812  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
813  # assembler=self.assembleCcd
814  # if self.config.doAssembleIsrExposures else None)
815  # if self.config.doFringe and
816  # self.fringe.checkFilter(inputData['ccdExposure'])
817  # else pipeBase.Struct(fringes=None))
818 
819  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
820 
821  def makeDatasetType(self, dsConfig):
822  return super().makeDatasetType(dsConfig)
823 
824  def readIsrData(self, dataRef, rawExposure):
825  """!Retrieve necessary frames for instrument signature removal.
826 
827  Pre-fetching all required ISR data products limits the IO
828  required by the ISR. Any conflict between the calibration data
829  available and that needed for ISR is also detected prior to
830  doing processing, allowing it to fail quickly.
831 
832  Parameters
833  ----------
834  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
835  Butler reference of the detector data to be processed
836  rawExposure : `afw.image.Exposure`
837  The raw exposure that will later be corrected with the
838  retrieved calibration data; should not be modified in this
839  method.
840 
841  Returns
842  -------
843  result : `lsst.pipe.base.Struct`
844  Result struct with components (which may be `None`):
845  - ``bias``: bias calibration frame (`afw.image.Exposure`)
846  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
847  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
848  - ``dark``: dark calibration frame (`afw.image.Exposure`)
849  - ``flat``: flat calibration frame (`afw.image.Exposure`)
850  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
851  - ``defects``: list of defects (`list`)
852  - ``fringes``: `lsst.pipe.base.Struct` with components:
853  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
854  - ``seed``: random seed derived from the ccdExposureId for random
855  number generator (`uint32`)
856  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
857  A ``TransmissionCurve`` that represents the throughput of the optics,
858  to be evaluated in focal-plane coordinates.
859  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
860  A ``TransmissionCurve`` that represents the throughput of the filter
861  itself, to be evaluated in focal-plane coordinates.
862  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
863  A ``TransmissionCurve`` that represents the throughput of the sensor
864  itself, to be evaluated in post-assembly trimmed detector coordinates.
865  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
866  A ``TransmissionCurve`` that represents the throughput of the
867  atmosphere, assumed to be spatially constant.
868 
869  Raises
870  ------
871  NotImplementedError :
872  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
873  """
874  ccd = rawExposure.getDetector()
875  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
876  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
877  if self.config.doBias else None)
878  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
879  linearizer = (dataRef.get("linearizer", immediate=True)
880  if self.doLinearize(ccd) else None)
881  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
882  if self.config.doCrosstalk else None)
883  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
884  if self.config.doDark else None)
885  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
886  if self.config.doFlat else None)
887 
888  brighterFatterKernel = None
889  if self.config.doBrighterFatter is True:
890 
891  # Use the new-style cp_pipe version of the kernel is it exists.
892  try:
893  brighterFatterKernel = dataRef.get("brighterFatterKernel")
894  except NoResults:
895  # Fall back to the old-style numpy-ndarray style kernel if necessary.
896  try:
897  brighterFatterKernel = dataRef.get("bfKernel")
898  except NoResults:
899  brighterFatterKernel = None
900  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
901  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
902  # this detector, or raise an error.
903  if self.config.brighterFatterLevel == 'DETECTOR':
904  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
905  else:
906  # TODO DM-15631 for implementing this
907  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
908 
909  defectList = (dataRef.get("defects")
910  if self.config.doDefect else None)
911  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
912  if self.config.doAssembleIsrExposures else None)
913  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
914  else pipeBase.Struct(fringes=None))
915 
916  if self.config.doAttachTransmissionCurve:
917  opticsTransmission = (dataRef.get("transmission_optics")
918  if self.config.doUseOpticsTransmission else None)
919  filterTransmission = (dataRef.get("transmission_filter")
920  if self.config.doUseFilterTransmission else None)
921  sensorTransmission = (dataRef.get("transmission_sensor")
922  if self.config.doUseSensorTransmission else None)
923  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
924  if self.config.doUseAtmosphereTransmission else None)
925  else:
926  opticsTransmission = None
927  filterTransmission = None
928  sensorTransmission = None
929  atmosphereTransmission = None
930 
931  # Struct should include only kwargs to run()
932  return pipeBase.Struct(bias=biasExposure,
933  linearizer=linearizer,
934  crosstalkSources=crosstalkSources,
935  dark=darkExposure,
936  flat=flatExposure,
937  bfKernel=brighterFatterKernel,
938  defects=defectList,
939  fringes=fringeStruct,
940  opticsTransmission=opticsTransmission,
941  filterTransmission=filterTransmission,
942  sensorTransmission=sensorTransmission,
943  atmosphereTransmission=atmosphereTransmission,
944  )
945 
946  @pipeBase.timeMethod
947  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
948  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
949  opticsTransmission=None, filterTransmission=None,
950  sensorTransmission=None, atmosphereTransmission=None,
951  detectorNum=None, isGen3=False
952  ):
953  """!Perform instrument signature removal on an exposure.
954 
955  Steps included in the ISR processing, in order performed, are:
956  - saturation and suspect pixel masking
957  - overscan subtraction
958  - CCD assembly of individual amplifiers
959  - bias subtraction
960  - variance image construction
961  - linearization of non-linear response
962  - crosstalk masking
963  - brighter-fatter correction
964  - dark subtraction
965  - fringe correction
966  - stray light subtraction
967  - flat correction
968  - masking of known defects and camera specific features
969  - vignette calculation
970  - appending transmission curve and distortion model
971 
972  Parameters
973  ----------
974  ccdExposure : `lsst.afw.image.Exposure`
975  The raw exposure that is to be run through ISR. The
976  exposure is modified by this method.
977  camera : `lsst.afw.cameraGeom.Camera`, optional
978  The camera geometry for this exposure. Used to select the
979  distortion model appropriate for this data.
980  bias : `lsst.afw.image.Exposure`, optional
981  Bias calibration frame.
982  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
983  Functor for linearization.
984  crosstalkSources : `list`, optional
985  List of possible crosstalk sources.
986  dark : `lsst.afw.image.Exposure`, optional
987  Dark calibration frame.
988  flat : `lsst.afw.image.Exposure`, optional
989  Flat calibration frame.
990  bfKernel : `numpy.ndarray`, optional
991  Brighter-fatter kernel.
992  defects : `list`, optional
993  List of defects.
994  fringes : `lsst.pipe.base.Struct`, optional
995  Struct containing the fringe correction data, with
996  elements:
997  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
998  - ``seed``: random seed derived from the ccdExposureId for random
999  number generator (`uint32`)
1000  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1001  A ``TransmissionCurve`` that represents the throughput of the optics,
1002  to be evaluated in focal-plane coordinates.
1003  filterTransmission : `lsst.afw.image.TransmissionCurve`
1004  A ``TransmissionCurve`` that represents the throughput of the filter
1005  itself, to be evaluated in focal-plane coordinates.
1006  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1007  A ``TransmissionCurve`` that represents the throughput of the sensor
1008  itself, to be evaluated in post-assembly trimmed detector coordinates.
1009  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1010  A ``TransmissionCurve`` that represents the throughput of the
1011  atmosphere, assumed to be spatially constant.
1012  detectorNum : `int`, optional
1013  The integer number for the detector to process.
1014  isGen3 : bool, optional
1015  Flag this call to run() as using the Gen3 butler environment.
1016 
1017  Returns
1018  -------
1019  result : `lsst.pipe.base.Struct`
1020  Result struct with component:
1021  - ``exposure`` : `afw.image.Exposure`
1022  The fully ISR corrected exposure.
1023  - ``outputExposure`` : `afw.image.Exposure`
1024  An alias for `exposure`
1025  - ``ossThumb`` : `numpy.ndarray`
1026  Thumbnail image of the exposure after overscan subtraction.
1027  - ``flattenedThumb`` : `numpy.ndarray`
1028  Thumbnail image of the exposure after flat-field correction.
1029 
1030  Raises
1031  ------
1032  RuntimeError
1033  Raised if a configuration option is set to True, but the
1034  required calibration data has not been specified.
1035 
1036  Notes
1037  -----
1038  The current processed exposure can be viewed by setting the
1039  appropriate lsstDebug entries in the `debug.display`
1040  dictionary. The names of these entries correspond to some of
1041  the IsrTaskConfig Boolean options, with the value denoting the
1042  frame to use. The exposure is shown inside the matching
1043  option check and after the processing of that step has
1044  finished. The steps with debug points are:
1045 
1046  doAssembleCcd
1047  doBias
1048  doCrosstalk
1049  doBrighterFatter
1050  doDark
1051  doFringe
1052  doStrayLight
1053  doFlat
1054 
1055  In addition, setting the "postISRCCD" entry displays the
1056  exposure after all ISR processing has finished.
1057 
1058  """
1059 
1060  if isGen3 is True:
1061  # Gen3 currently cannot automatically do configuration overrides.
1062  # DM-15257 looks to discuss this issue.
1063 
1064  self.config.doFringe = False
1065 
1066  # Configure input exposures;
1067  if detectorNum is None:
1068  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1069 
1070  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1071  bias = self.ensureExposure(bias, camera, detectorNum)
1072  dark = self.ensureExposure(dark, camera, detectorNum)
1073  flat = self.ensureExposure(flat, camera, detectorNum)
1074  else:
1075  if isinstance(ccdExposure, ButlerDataRef):
1076  return self.runDataRef(ccdExposure)
1077 
1078  ccd = ccdExposure.getDetector()
1079 
1080  if not ccd:
1081  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1082  ccd = [FakeAmp(ccdExposure, self.config)]
1083 
1084  # Validate Input
1085  if self.config.doBias and bias is None:
1086  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1087  if self.doLinearize(ccd) and linearizer is None:
1088  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1089  if self.config.doBrighterFatter and bfKernel is None:
1090  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1091  if self.config.doDark and dark is None:
1092  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1093  if fringes is None:
1094  fringes = pipeBase.Struct(fringes=None)
1095  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1096  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1097  if self.config.doFlat and flat is None:
1098  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1099  if self.config.doDefect and defects is None:
1100  raise RuntimeError("Must supply defects if config.doDefect=True.")
1101  if self.config.doAddDistortionModel and camera is None:
1102  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1103 
1104  # Begin ISR processing.
1105  if self.config.doConvertIntToFloat:
1106  self.log.info("Converting exposure to floating point values")
1107  ccdExposure = self.convertIntToFloat(ccdExposure)
1108 
1109  # Amplifier level processing.
1110  overscans = []
1111  for amp in ccd:
1112  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1113  if ccdExposure.getBBox().contains(amp.getBBox()):
1114  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1115  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1116 
1117  if self.config.doOverscan and not badAmp:
1118  # Overscan correction on amp-by-amp basis.
1119  overscanResults = self.overscanCorrection(ccdExposure, amp)
1120  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1121  if self.config.qa is not None and self.config.qa.saveStats is True:
1122  if isinstance(overscanResults.overscanFit, float):
1123  qaMedian = overscanResults.overscanFit
1124  qaStdev = float("NaN")
1125  else:
1126  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1127  afwMath.MEDIAN | afwMath.STDEVCLIP)
1128  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1129  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1130 
1131  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1132  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1133  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1134  (amp.getName(), qaMedian, qaStdev))
1135  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1136  else:
1137  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1138  overscanResults = None
1139 
1140  overscans.append(overscanResults if overscanResults is not None else None)
1141  else:
1142  self.log.info("Skipped OSCAN")
1143 
1144  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1145  self.log.info("Applying crosstalk correction.")
1146  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1147  self.debugView(ccdExposure, "doCrosstalk")
1148 
1149  if self.config.doAssembleCcd:
1150  self.log.info("Assembling CCD from amplifiers")
1151  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1152 
1153  if self.config.expectWcs and not ccdExposure.getWcs():
1154  self.log.warn("No WCS found in input exposure")
1155  self.debugView(ccdExposure, "doAssembleCcd")
1156 
1157  ossThumb = None
1158  if self.config.qa.doThumbnailOss:
1159  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1160 
1161  if self.config.doBias:
1162  self.log.info("Applying bias correction.")
1163  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1164  trimToFit=self.config.doTrimToMatchCalib)
1165  self.debugView(ccdExposure, "doBias")
1166 
1167  if self.config.doVariance:
1168  for amp, overscanResults in zip(ccd, overscans):
1169  if ccdExposure.getBBox().contains(amp.getBBox()):
1170  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1171  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1172  if overscanResults is not None:
1173  self.updateVariance(ampExposure, amp,
1174  overscanImage=overscanResults.overscanImage)
1175  else:
1176  self.updateVariance(ampExposure, amp,
1177  overscanImage=None)
1178  if self.config.qa is not None and self.config.qa.saveStats is True:
1179  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1180  afwMath.MEDIAN | afwMath.STDEVCLIP)
1181  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1182  qaStats.getValue(afwMath.MEDIAN))
1183  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1184  qaStats.getValue(afwMath.STDEVCLIP))
1185  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1186  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1187  qaStats.getValue(afwMath.STDEVCLIP)))
1188 
1189  if self.doLinearize(ccd):
1190  self.log.info("Applying linearizer.")
1191  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1192 
1193  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1194  self.log.info("Applying crosstalk correction.")
1195  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1196  self.debugView(ccdExposure, "doCrosstalk")
1197 
1198  if self.config.doWidenSaturationTrails:
1199  self.log.info("Widening saturation trails.")
1200  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1201 
1202  interpolationDone = False
1203  if self.config.doBrighterFatter:
1204  # We need to apply flats and darks before we can interpolate, and we
1205  # need to interpolate before we do B-F, but we do B-F without the
1206  # flats and darks applied so we can work in units of electrons or holes.
1207  # This context manager applies and then removes the darks and flats.
1208  with self.flatContext(ccdExposure, flat, dark):
1209  if self.config.doDefect:
1210  self.maskAndInterpDefect(ccdExposure, defects)
1211 
1212  if self.config.doSaturationInterpolation:
1213  self.saturationInterpolation(ccdExposure)
1214 
1215  self.maskAndInterpNan(ccdExposure)
1216  interpolationDone = True
1217 
1218  self.log.info("Applying brighter fatter correction.")
1219  isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1220  self.config.brighterFatterMaxIter,
1221  self.config.brighterFatterThreshold,
1222  self.config.brighterFatterApplyGain,
1223  )
1224  self.debugView(ccdExposure, "doBrighterFatter")
1225 
1226  if self.config.doDark:
1227  self.log.info("Applying dark correction.")
1228  self.darkCorrection(ccdExposure, dark)
1229  self.debugView(ccdExposure, "doDark")
1230 
1231  if self.config.doFringe and not self.config.fringeAfterFlat:
1232  self.log.info("Applying fringe correction before flat.")
1233  self.fringe.run(ccdExposure, **fringes.getDict())
1234  self.debugView(ccdExposure, "doFringe")
1235 
1236  if self.config.doStrayLight:
1237  self.log.info("Applying stray light correction.")
1238  self.strayLight.run(ccdExposure)
1239  self.debugView(ccdExposure, "doStrayLight")
1240 
1241  if self.config.doFlat:
1242  self.log.info("Applying flat correction.")
1243  self.flatCorrection(ccdExposure, flat)
1244  self.debugView(ccdExposure, "doFlat")
1245 
1246  if self.config.doApplyGains:
1247  self.log.info("Applying gain correction instead of flat.")
1248  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1249 
1250  if self.config.doDefect and not interpolationDone:
1251  self.log.info("Masking and interpolating defects.")
1252  self.maskAndInterpDefect(ccdExposure, defects)
1253 
1254  if self.config.doSaturationInterpolation and not interpolationDone:
1255  self.log.info("Interpolating saturated pixels.")
1256  self.saturationInterpolation(ccdExposure)
1257 
1258  if self.config.doNanInterpAfterFlat or not interpolationDone:
1259  self.log.info("Masking and interpolating NAN value pixels.")
1260  self.maskAndInterpNan(ccdExposure)
1261 
1262  if self.config.doFringe and self.config.fringeAfterFlat:
1263  self.log.info("Applying fringe correction after flat.")
1264  self.fringe.run(ccdExposure, **fringes.getDict())
1265 
1266  if self.config.doSetBadRegions:
1267  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1268  if badPixelCount > 0:
1269  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1270 
1271  flattenedThumb = None
1272  if self.config.qa.doThumbnailFlattened:
1273  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1274 
1275  if self.config.doCameraSpecificMasking:
1276  self.log.info("Masking regions for camera specific reasons.")
1277  self.masking.run(ccdExposure)
1278 
1279  self.roughZeroPoint(ccdExposure)
1280 
1281  if self.config.doVignette:
1282  self.log.info("Constructing Vignette polygon.")
1283  self.vignettePolygon = self.vignette.run(ccdExposure)
1284 
1285  if self.config.vignette.doWriteVignettePolygon:
1286  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1287 
1288  if self.config.doAttachTransmissionCurve:
1289  self.log.info("Adding transmission curves.")
1290  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1291  filterTransmission=filterTransmission,
1292  sensorTransmission=sensorTransmission,
1293  atmosphereTransmission=atmosphereTransmission)
1294 
1295  if self.config.doAddDistortionModel:
1296  self.log.info("Adding a distortion model to the WCS.")
1297  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1298 
1299  if self.config.doMeasureBackground:
1300  self.log.info("Measuring background level:")
1301  self.measureBackground(ccdExposure, self.config.qa)
1302 
1303  if self.config.qa is not None and self.config.qa.saveStats is True:
1304  for amp in ccd:
1305  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1306  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1307  afwMath.MEDIAN | afwMath.STDEVCLIP)
1308  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1309  qaStats.getValue(afwMath.MEDIAN))
1310  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1311  qaStats.getValue(afwMath.STDEVCLIP))
1312  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1313  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1314  qaStats.getValue(afwMath.STDEVCLIP)))
1315 
1316  self.debugView(ccdExposure, "postISRCCD")
1317 
1318  return pipeBase.Struct(
1319  exposure=ccdExposure,
1320  ossThumb=ossThumb,
1321  flattenedThumb=flattenedThumb,
1322 
1323  outputExposure=ccdExposure,
1324  outputOssThumbnail=ossThumb,
1325  outputFlattenedThumbnail=flattenedThumb,
1326  )
1327 
1328  @pipeBase.timeMethod
1329  def runDataRef(self, sensorRef):
1330  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1331 
1332  This method contains the `CmdLineTask` interface to the ISR
1333  processing. All IO is handled here, freeing the `run()` method
1334  to manage only pixel-level calculations. The steps performed
1335  are:
1336  - Read in necessary detrending/isr/calibration data.
1337  - Process raw exposure in `run()`.
1338  - Persist the ISR-corrected exposure as "postISRCCD" if
1339  config.doWrite=True.
1340 
1341  Parameters
1342  ----------
1343  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1344  DataRef of the detector data to be processed
1345 
1346  Returns
1347  -------
1348  result : `lsst.pipe.base.Struct`
1349  Result struct with component:
1350  - ``exposure`` : `afw.image.Exposure`
1351  The fully ISR corrected exposure.
1352 
1353  Raises
1354  ------
1355  RuntimeError
1356  Raised if a configuration option is set to True, but the
1357  required calibration data does not exist.
1358 
1359  """
1360  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1361 
1362  ccdExposure = sensorRef.get(self.config.datasetType)
1363 
1364  camera = sensorRef.get("camera")
1365  if camera is None and self.config.doAddDistortionModel:
1366  raise RuntimeError("config.doAddDistortionModel is True "
1367  "but could not get a camera from the butler")
1368  isrData = self.readIsrData(sensorRef, ccdExposure)
1369 
1370  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1371 
1372  if self.config.doWrite:
1373  sensorRef.put(result.exposure, "postISRCCD")
1374  if result.ossThumb is not None:
1375  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1376  if result.flattenedThumb is not None:
1377  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1378 
1379  return result
1380 
1381  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1382  """!Retrieve a calibration dataset for removing instrument signature.
1383 
1384  Parameters
1385  ----------
1386 
1387  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1388  DataRef of the detector data to find calibration datasets
1389  for.
1390  datasetType : `str`
1391  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1392  immediate : `Bool`
1393  If True, disable butler proxies to enable error handling
1394  within this routine.
1395 
1396  Returns
1397  -------
1398  exposure : `lsst.afw.image.Exposure`
1399  Requested calibration frame.
1400 
1401  Raises
1402  ------
1403  RuntimeError
1404  Raised if no matching calibration frame can be found.
1405  """
1406  try:
1407  exp = dataRef.get(datasetType, immediate=immediate)
1408  except Exception as exc1:
1409  if not self.config.fallbackFilterName:
1410  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1411  try:
1412  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1413  except Exception as exc2:
1414  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1415  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1416  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1417 
1418  if self.config.doAssembleIsrExposures:
1419  exp = self.assembleCcd.assembleCcd(exp)
1420  return exp
1421 
1422  def ensureExposure(self, inputExp, camera, detectorNum):
1423  """Ensure that the data returned by Butler is a fully constructed exposure.
1424 
1425  ISR requires exposure-level image data for historical reasons, so if we did
1426  not recieve that from Butler, construct it from what we have, modifying the
1427  input in place.
1428 
1429  Parameters
1430  ----------
1431  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1432  `lsst.afw.image.ImageF`
1433  The input data structure obtained from Butler.
1434  camera : `lsst.afw.cameraGeom.camera`
1435  The camera associated with the image. Used to find the appropriate
1436  detector.
1437  detectorNum : `int`
1438  The detector this exposure should match.
1439 
1440  Returns
1441  -------
1442  inputExp : `lsst.afw.image.Exposure`
1443  The re-constructed exposure, with appropriate detector parameters.
1444 
1445  Raises
1446  ------
1447  TypeError
1448  Raised if the input data cannot be used to construct an exposure.
1449  """
1450  if isinstance(inputExp, afwImage.DecoratedImageU):
1451  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1452  elif isinstance(inputExp, afwImage.ImageF):
1453  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1454  elif isinstance(inputExp, afwImage.MaskedImageF):
1455  inputExp = afwImage.makeExposure(inputExp)
1456  elif isinstance(inputExp, afwImage.Exposure):
1457  pass
1458  else:
1459  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1460 
1461  if inputExp.getDetector() is None:
1462  inputExp.setDetector(camera[detectorNum])
1463 
1464  return inputExp
1465 
1466  def convertIntToFloat(self, exposure):
1467  """Convert exposure image from uint16 to float.
1468 
1469  If the exposure does not need to be converted, the input is
1470  immediately returned. For exposures that are converted to use
1471  floating point pixels, the variance is set to unity and the
1472  mask to zero.
1473 
1474  Parameters
1475  ----------
1476  exposure : `lsst.afw.image.Exposure`
1477  The raw exposure to be converted.
1478 
1479  Returns
1480  -------
1481  newexposure : `lsst.afw.image.Exposure`
1482  The input ``exposure``, converted to floating point pixels.
1483 
1484  Raises
1485  ------
1486  RuntimeError
1487  Raised if the exposure type cannot be converted to float.
1488 
1489  """
1490  if isinstance(exposure, afwImage.ExposureF):
1491  # Nothing to be done
1492  return exposure
1493  if not hasattr(exposure, "convertF"):
1494  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1495 
1496  newexposure = exposure.convertF()
1497  newexposure.variance[:] = 1
1498  newexposure.mask[:] = 0x0
1499 
1500  return newexposure
1501 
1502  def maskAmplifier(self, ccdExposure, amp, defects):
1503  """Identify bad amplifiers, saturated and suspect pixels.
1504 
1505  Parameters
1506  ----------
1507  ccdExposure : `lsst.afw.image.Exposure`
1508  Input exposure to be masked.
1509  amp : `lsst.afw.table.AmpInfoCatalog`
1510  Catalog of parameters defining the amplifier on this
1511  exposure to mask.
1512  defects : `list`
1513  List of defects. Used to determine if the entire
1514  amplifier is bad.
1515 
1516  Returns
1517  -------
1518  badAmp : `Bool`
1519  If this is true, the entire amplifier area is covered by
1520  defects and unusable.
1521 
1522  """
1523  maskedImage = ccdExposure.getMaskedImage()
1524 
1525  badAmp = False
1526 
1527  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1528  # comparison with current defects definition.
1529  if defects is not None:
1530  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1531 
1532  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1533  # association with pixels in current ccdExposure).
1534  if badAmp:
1535  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1536  afwImage.PARENT)
1537  maskView = dataView.getMask()
1538  maskView |= maskView.getPlaneBitMask("BAD")
1539  del maskView
1540  return badAmp
1541 
1542  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1543  # Saturation and suspect pixels can be masked now, though.
1544  limits = dict()
1545  if self.config.doSaturation and not badAmp:
1546  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1547  if self.config.doSuspect and not badAmp:
1548  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1549 
1550  for maskName, maskThreshold in limits.items():
1551  if not math.isnan(maskThreshold):
1552  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1553  isrFunctions.makeThresholdMask(
1554  maskedImage=dataView,
1555  threshold=maskThreshold,
1556  growFootprints=0,
1557  maskName=maskName
1558  )
1559 
1560  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1561  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1562  afwImage.PARENT)
1563  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1564  self.config.suspectMaskName])
1565  if numpy.all(maskView.getArray() & maskVal > 0):
1566  badAmp = True
1567 
1568  return badAmp
1569 
1570  def overscanCorrection(self, ccdExposure, amp):
1571  """Apply overscan correction in place.
1572 
1573  This method does initial pixel rejection of the overscan
1574  region. The overscan can also be optionally segmented to
1575  allow for discontinuous overscan responses to be fit
1576  separately. The actual overscan subtraction is performed by
1577  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1578  which is called here after the amplifier is preprocessed.
1579 
1580  Parameters
1581  ----------
1582  ccdExposure : `lsst.afw.image.Exposure`
1583  Exposure to have overscan correction performed.
1584  amp : `lsst.afw.table.AmpInfoCatalog`
1585  The amplifier to consider while correcting the overscan.
1586 
1587  Returns
1588  -------
1589  overscanResults : `lsst.pipe.base.Struct`
1590  Result struct with components:
1591  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1592  Value or fit subtracted from the amplifier image data.
1593  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1594  Value or fit subtracted from the overscan image data.
1595  - ``overscanImage`` : `lsst.afw.image.Image`
1596  Image of the overscan region with the overscan
1597  correction applied. This quantity is used to estimate
1598  the amplifier read noise empirically.
1599 
1600  Raises
1601  ------
1602  RuntimeError
1603  Raised if the ``amp`` does not contain raw pixel information.
1604 
1605  See Also
1606  --------
1607  lsst.ip.isr.isrFunctions.overscanCorrection
1608  """
1609  if not amp.getHasRawInfo():
1610  raise RuntimeError("This method must be executed on an amp with raw information.")
1611 
1612  if amp.getRawHorizontalOverscanBBox().isEmpty():
1613  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1614  return None
1615 
1616  statControl = afwMath.StatisticsControl()
1617  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1618 
1619  # Determine the bounding boxes
1620  dataBBox = amp.getRawDataBBox()
1621  oscanBBox = amp.getRawHorizontalOverscanBBox()
1622  dx0 = 0
1623  dx1 = 0
1624 
1625  prescanBBox = amp.getRawPrescanBBox()
1626  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1627  dx0 += self.config.overscanNumLeadingColumnsToSkip
1628  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1629  else:
1630  dx0 += self.config.overscanNumTrailingColumnsToSkip
1631  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1632 
1633  # Determine if we need to work on subregions of the amplifier and overscan.
1634  imageBBoxes = []
1635  overscanBBoxes = []
1636 
1637  if ((self.config.overscanBiasJump and
1638  self.config.overscanBiasJumpLocation) and
1639  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1640  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1641  self.config.overscanBiasJumpDevices)):
1642  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1643  yLower = self.config.overscanBiasJumpLocation
1644  yUpper = dataBBox.getHeight() - yLower
1645  else:
1646  yUpper = self.config.overscanBiasJumpLocation
1647  yLower = dataBBox.getHeight() - yUpper
1648 
1649  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1650  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1651  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1652  afwGeom.Extent2I(dx0, 0),
1653  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1654  yLower)))
1655 
1656  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1657  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1658  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, yLower),
1659  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1660  yUpper)))
1661  else:
1662  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1663  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1664  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(dx0, 0),
1665  afwGeom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1666  oscanBBox.getHeight())))
1667 
1668  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1669  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1670  ampImage = ccdExposure.maskedImage[imageBBox]
1671  overscanImage = ccdExposure.maskedImage[overscanBBox]
1672 
1673  overscanArray = overscanImage.image.array
1674  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1675  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1676  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1677 
1678  statControl = afwMath.StatisticsControl()
1679  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1680 
1681  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1682  overscanImage=overscanImage,
1683  fitType=self.config.overscanFitType,
1684  order=self.config.overscanOrder,
1685  collapseRej=self.config.overscanNumSigmaClip,
1686  statControl=statControl,
1687  overscanIsInt=self.config.overscanIsInt
1688  )
1689 
1690  # Measure average overscan levels and record them in the metadata
1691  levelStat = afwMath.MEDIAN
1692  sigmaStat = afwMath.STDEVCLIP
1693 
1694  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1695  self.config.qa.flatness.nIter)
1696  metadata = ccdExposure.getMetadata()
1697  ampNum = amp.getName()
1698  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1699  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1700  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1701  else:
1702  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1703  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1704  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1705 
1706  return overscanResults
1707 
1708  def updateVariance(self, ampExposure, amp, overscanImage=None):
1709  """Set the variance plane using the amplifier gain and read noise
1710 
1711  The read noise is calculated from the ``overscanImage`` if the
1712  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1713  the value from the amplifier data is used.
1714 
1715  Parameters
1716  ----------
1717  ampExposure : `lsst.afw.image.Exposure`
1718  Exposure to process.
1719  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1720  Amplifier detector data.
1721  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1722  Image of overscan, required only for empirical read noise.
1723 
1724  See also
1725  --------
1726  lsst.ip.isr.isrFunctions.updateVariance
1727  """
1728  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1729  gain = amp.getGain()
1730 
1731  if math.isnan(gain):
1732  gain = 1.0
1733  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1734  elif gain <= 0:
1735  patchedGain = 1.0
1736  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1737  (amp.getName(), gain, patchedGain))
1738  gain = patchedGain
1739 
1740  if self.config.doEmpiricalReadNoise and overscanImage is None:
1741  self.log.info("Overscan is none for EmpiricalReadNoise")
1742 
1743  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1744  stats = afwMath.StatisticsControl()
1745  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1746  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1747  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1748  else:
1749  readNoise = amp.getReadNoise()
1750 
1751  isrFunctions.updateVariance(
1752  maskedImage=ampExposure.getMaskedImage(),
1753  gain=gain,
1754  readNoise=readNoise,
1755  )
1756 
1757  def darkCorrection(self, exposure, darkExposure, invert=False):
1758  """!Apply dark correction in place.
1759 
1760  Parameters
1761  ----------
1762  exposure : `lsst.afw.image.Exposure`
1763  Exposure to process.
1764  darkExposure : `lsst.afw.image.Exposure`
1765  Dark exposure of the same size as ``exposure``.
1766  invert : `Bool`, optional
1767  If True, re-add the dark to an already corrected image.
1768 
1769  Raises
1770  ------
1771  RuntimeError
1772  Raised if either ``exposure`` or ``darkExposure`` do not
1773  have their dark time defined.
1774 
1775  See Also
1776  --------
1777  lsst.ip.isr.isrFunctions.darkCorrection
1778  """
1779  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1780  if math.isnan(expScale):
1781  raise RuntimeError("Exposure darktime is NAN")
1782  if darkExposure.getInfo().getVisitInfo() is not None:
1783  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1784  else:
1785  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1786  # so getDarkTime() does not exist.
1787  darkScale = 1.0
1788 
1789  if math.isnan(darkScale):
1790  raise RuntimeError("Dark calib darktime is NAN")
1791  isrFunctions.darkCorrection(
1792  maskedImage=exposure.getMaskedImage(),
1793  darkMaskedImage=darkExposure.getMaskedImage(),
1794  expScale=expScale,
1795  darkScale=darkScale,
1796  invert=invert,
1797  trimToFit=self.config.doTrimToMatchCalib
1798  )
1799 
1800  def doLinearize(self, detector):
1801  """!Check if linearization is needed for the detector cameraGeom.
1802 
1803  Checks config.doLinearize and the linearity type of the first
1804  amplifier.
1805 
1806  Parameters
1807  ----------
1808  detector : `lsst.afw.cameraGeom.Detector`
1809  Detector to get linearity type from.
1810 
1811  Returns
1812  -------
1813  doLinearize : `Bool`
1814  If True, linearization should be performed.
1815  """
1816  return self.config.doLinearize and \
1817  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1818 
1819  def flatCorrection(self, exposure, flatExposure, invert=False):
1820  """!Apply flat correction in place.
1821 
1822  Parameters
1823  ----------
1824  exposure : `lsst.afw.image.Exposure`
1825  Exposure to process.
1826  flatExposure : `lsst.afw.image.Exposure`
1827  Flat exposure of the same size as ``exposure``.
1828  invert : `Bool`, optional
1829  If True, unflatten an already flattened image.
1830 
1831  See Also
1832  --------
1833  lsst.ip.isr.isrFunctions.flatCorrection
1834  """
1835  isrFunctions.flatCorrection(
1836  maskedImage=exposure.getMaskedImage(),
1837  flatMaskedImage=flatExposure.getMaskedImage(),
1838  scalingType=self.config.flatScalingType,
1839  userScale=self.config.flatUserScale,
1840  invert=invert,
1841  trimToFit=self.config.doTrimToMatchCalib
1842  )
1843 
1844  def saturationDetection(self, exposure, amp):
1845  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1846 
1847  Parameters
1848  ----------
1849  exposure : `lsst.afw.image.Exposure`
1850  Exposure to process. Only the amplifier DataSec is processed.
1851  amp : `lsst.afw.table.AmpInfoCatalog`
1852  Amplifier detector data.
1853 
1854  See Also
1855  --------
1856  lsst.ip.isr.isrFunctions.makeThresholdMask
1857  """
1858  if not math.isnan(amp.getSaturation()):
1859  maskedImage = exposure.getMaskedImage()
1860  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1861  isrFunctions.makeThresholdMask(
1862  maskedImage=dataView,
1863  threshold=amp.getSaturation(),
1864  growFootprints=0,
1865  maskName=self.config.saturatedMaskName,
1866  )
1867 
1868  def saturationInterpolation(self, ccdExposure):
1869  """!Interpolate over saturated pixels, in place.
1870 
1871  This method should be called after `saturationDetection`, to
1872  ensure that the saturated pixels have been identified in the
1873  SAT mask. It should also be called after `assembleCcd`, since
1874  saturated regions may cross amplifier boundaries.
1875 
1876  Parameters
1877  ----------
1878  exposure : `lsst.afw.image.Exposure`
1879  Exposure to process.
1880 
1881  See Also
1882  --------
1883  lsst.ip.isr.isrTask.saturationDetection
1884  lsst.ip.isr.isrFunctions.interpolateFromMask
1885  """
1886  isrFunctions.interpolateFromMask(
1887  maskedImage=ccdExposure.getMaskedImage(),
1888  fwhm=self.config.fwhm,
1889  growFootprints=self.config.growSaturationFootprintSize,
1890  maskName=self.config.saturatedMaskName,
1891  )
1892 
1893  def suspectDetection(self, exposure, amp):
1894  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1895 
1896  Parameters
1897  ----------
1898  exposure : `lsst.afw.image.Exposure`
1899  Exposure to process. Only the amplifier DataSec is processed.
1900  amp : `lsst.afw.table.AmpInfoCatalog`
1901  Amplifier detector data.
1902 
1903  See Also
1904  --------
1905  lsst.ip.isr.isrFunctions.makeThresholdMask
1906 
1907  Notes
1908  -----
1909  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1910  This is intended to indicate pixels that may be affected by unknown systematics;
1911  for example if non-linearity corrections above a certain level are unstable
1912  then that would be a useful value for suspectLevel. A value of `nan` indicates
1913  that no such level exists and no pixels are to be masked as suspicious.
1914  """
1915  suspectLevel = amp.getSuspectLevel()
1916  if math.isnan(suspectLevel):
1917  return
1918 
1919  maskedImage = exposure.getMaskedImage()
1920  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1921  isrFunctions.makeThresholdMask(
1922  maskedImage=dataView,
1923  threshold=suspectLevel,
1924  growFootprints=0,
1925  maskName=self.config.suspectMaskName,
1926  )
1927 
1928  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1929  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1930 
1931  Parameters
1932  ----------
1933  ccdExposure : `lsst.afw.image.Exposure`
1934  Exposure to process.
1935  defectBaseList : `List`
1936  List of defects to mask and interpolate.
1937 
1938  Notes
1939  -----
1940  Call this after CCD assembly, since defects may cross amplifier boundaries.
1941  """
1942  maskedImage = ccdExposure.getMaskedImage()
1943  defectList = []
1944  for d in defectBaseList:
1945  bbox = d.getBBox()
1946  nd = measAlg.Defect(bbox)
1947  defectList.append(nd)
1948  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1949  isrFunctions.interpolateDefectList(
1950  maskedImage=maskedImage,
1951  defectList=defectList,
1952  fwhm=self.config.fwhm,
1953  )
1954 
1955  if self.config.numEdgeSuspect > 0:
1956  goodBBox = maskedImage.getBBox()
1957  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1958  goodBBox.grow(-self.config.numEdgeSuspect)
1959  # Mask pixels outside goodBBox as SUSPECT
1960  SourceDetectionTask.setEdgeBits(
1961  maskedImage,
1962  goodBBox,
1963  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1964  )
1965 
1966  def maskAndInterpNan(self, exposure):
1967  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1968 
1969  Parameters
1970  ----------
1971  exposure : `lsst.afw.image.Exposure`
1972  Exposure to process.
1973 
1974  Notes
1975  -----
1976  We mask and interpolate over all NaNs, including those
1977  that are masked with other bits (because those may or may
1978  not be interpolated over later, and we want to remove all
1979  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1980  is used to preserve the historical name.
1981  """
1982  maskedImage = exposure.getMaskedImage()
1983 
1984  # Find and mask NaNs
1985  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1986  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1987  numNans = maskNans(maskedImage, maskVal)
1988  self.metadata.set("NUMNANS", numNans)
1989 
1990  # Interpolate over these previously-unmasked NaNs
1991  if numNans > 0:
1992  self.log.warn("There were %i unmasked NaNs", numNans)
1993  nanDefectList = isrFunctions.getDefectListFromMask(
1994  maskedImage=maskedImage,
1995  maskName='UNMASKEDNAN',
1996  )
1997  isrFunctions.interpolateDefectList(
1998  maskedImage=exposure.getMaskedImage(),
1999  defectList=nanDefectList,
2000  fwhm=self.config.fwhm,
2001  )
2002 
2003  def measureBackground(self, exposure, IsrQaConfig=None):
2004  """Measure the image background in subgrids, for quality control purposes.
2005 
2006  Parameters
2007  ----------
2008  exposure : `lsst.afw.image.Exposure`
2009  Exposure to process.
2010  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2011  Configuration object containing parameters on which background
2012  statistics and subgrids to use.
2013  """
2014  if IsrQaConfig is not None:
2015  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2016  IsrQaConfig.flatness.nIter)
2017  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2018  statsControl.setAndMask(maskVal)
2019  maskedImage = exposure.getMaskedImage()
2020  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2021  skyLevel = stats.getValue(afwMath.MEDIAN)
2022  skySigma = stats.getValue(afwMath.STDEVCLIP)
2023  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2024  metadata = exposure.getMetadata()
2025  metadata.set('SKYLEVEL', skyLevel)
2026  metadata.set('SKYSIGMA', skySigma)
2027 
2028  # calcluating flatlevel over the subgrids
2029  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2030  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2031  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2032  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2033  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2034  skyLevels = numpy.zeros((nX, nY))
2035 
2036  for j in range(nY):
2037  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2038  for i in range(nX):
2039  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2040 
2041  xLLC = xc - meshXHalf
2042  yLLC = yc - meshYHalf
2043  xURC = xc + meshXHalf - 1
2044  yURC = yc + meshYHalf - 1
2045 
2046  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2047  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2048 
2049  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2050 
2051  good = numpy.where(numpy.isfinite(skyLevels))
2052  skyMedian = numpy.median(skyLevels[good])
2053  flatness = (skyLevels[good] - skyMedian) / skyMedian
2054  flatness_rms = numpy.std(flatness)
2055  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2056 
2057  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2058  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2059  (nX, nY, flatness_pp, flatness_rms))
2060 
2061  metadata.set('FLATNESS_PP', float(flatness_pp))
2062  metadata.set('FLATNESS_RMS', float(flatness_rms))
2063  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2064  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2065  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2066 
2067  def roughZeroPoint(self, exposure):
2068  """Set an approximate magnitude zero point for the exposure.
2069 
2070  Parameters
2071  ----------
2072  exposure : `lsst.afw.image.Exposure`
2073  Exposure to process.
2074  """
2075  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2076  if filterName in self.config.fluxMag0T1:
2077  fluxMag0 = self.config.fluxMag0T1[filterName]
2078  else:
2079  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2080  fluxMag0 = self.config.defaultFluxMag0T1
2081 
2082  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2083  if not expTime > 0: # handle NaN as well as <= 0
2084  self.log.warn("Non-positive exposure time; skipping rough zero point")
2085  return
2086 
2087  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2088  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2089 
2090  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2091  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2092 
2093  Parameters
2094  ----------
2095  ccdExposure : `lsst.afw.image.Exposure`
2096  Exposure to process.
2097  fpPolygon : `lsst.afw.geom.Polygon`
2098  Polygon in focal plane coordinates.
2099  """
2100  # Get ccd corners in focal plane coordinates
2101  ccd = ccdExposure.getDetector()
2102  fpCorners = ccd.getCorners(FOCAL_PLANE)
2103  ccdPolygon = Polygon(fpCorners)
2104 
2105  # Get intersection of ccd corners with fpPolygon
2106  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2107 
2108  # Transform back to pixel positions and build new polygon
2109  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2110  validPolygon = Polygon(ccdPoints)
2111  ccdExposure.getInfo().setValidPolygon(validPolygon)
2112 
2113  @contextmanager
2114  def flatContext(self, exp, flat, dark=None):
2115  """Context manager that applies and removes flats and darks,
2116  if the task is configured to apply them.
2117 
2118  Parameters
2119  ----------
2120  exp : `lsst.afw.image.Exposure`
2121  Exposure to process.
2122  flat : `lsst.afw.image.Exposure`
2123  Flat exposure the same size as ``exp``.
2124  dark : `lsst.afw.image.Exposure`, optional
2125  Dark exposure the same size as ``exp``.
2126 
2127  Yields
2128  ------
2129  exp : `lsst.afw.image.Exposure`
2130  The flat and dark corrected exposure.
2131  """
2132  if self.config.doDark and dark is not None:
2133  self.darkCorrection(exp, dark)
2134  if self.config.doFlat:
2135  self.flatCorrection(exp, flat)
2136  try:
2137  yield exp
2138  finally:
2139  if self.config.doFlat:
2140  self.flatCorrection(exp, flat, invert=True)
2141  if self.config.doDark and dark is not None:
2142  self.darkCorrection(exp, dark, invert=True)
2143 
2144  def debugView(self, exposure, stepname):
2145  """Utility function to examine ISR exposure at different stages.
2146 
2147  Parameters
2148  ----------
2149  exposure : `lsst.afw.image.Exposure`
2150  Exposure to view.
2151  stepname : `str`
2152  State of processing to view.
2153  """
2154  frame = getDebugFrame(self._display, stepname)
2155  if frame:
2156  display = getDisplay(frame)
2157  display.scale('asinh', 'zscale')
2158  display.mtv(exposure)
2159 
2160 
2161 class FakeAmp(object):
2162  """A Detector-like object that supports returning gain and saturation level
2163 
2164  This is used when the input exposure does not have a detector.
2165 
2166  Parameters
2167  ----------
2168  exposure : `lsst.afw.image.Exposure`
2169  Exposure to generate a fake amplifier for.
2170  config : `lsst.ip.isr.isrTaskConfig`
2171  Configuration to apply to the fake amplifier.
2172  """
2173 
2174  def __init__(self, exposure, config):
2175  self._bbox = exposure.getBBox(afwImage.LOCAL)
2176  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
2177  self._gain = config.gain
2178  self._readNoise = config.readNoise
2179  self._saturation = config.saturation
2180 
2181  def getBBox(self):
2182  return self._bbox
2183 
2184  def getRawBBox(self):
2185  return self._bbox
2186 
2187  def getHasRawInfo(self):
2188  return True # but see getRawHorizontalOverscanBBox()
2189 
2191  return self._RawHorizontalOverscanBBox
2192 
2193  def getGain(self):
2194  return self._gain
2195 
2196  def getReadNoise(self):
2197  return self._readNoise
2198 
2199  def getSaturation(self):
2200  return self._saturation
2201 
2202  def getSuspectLevel(self):
2203  return float("NaN")
2204 
2205 
2206 class RunIsrConfig(pexConfig.Config):
2207  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2208 
2209 
2210 class RunIsrTask(pipeBase.CmdLineTask):
2211  """Task to wrap the default IsrTask to allow it to be retargeted.
2212 
2213  The standard IsrTask can be called directly from a command line
2214  program, but doing so removes the ability of the task to be
2215  retargeted. As most cameras override some set of the IsrTask
2216  methods, this would remove those data-specific methods in the
2217  output post-ISR images. This wrapping class fixes the issue,
2218  allowing identical post-ISR images to be generated by both the
2219  processCcd and isrTask code.
2220  """
2221  ConfigClass = RunIsrConfig
2222  _DefaultName = "runIsr"
2223 
2224  def __init__(self, *args, **kwargs):
2225  super().__init__(*args, **kwargs)
2226  self.makeSubtask("isr")
2227 
2228  def runDataRef(self, dataRef):
2229  """
2230  Parameters
2231  ----------
2232  dataRef : `lsst.daf.persistence.ButlerDataRef`
2233  data reference of the detector data to be processed
2234 
2235  Returns
2236  -------
2237  result : `pipeBase.Struct`
2238  Result struct with component:
2239 
2240  - exposure : `lsst.afw.image.Exposure`
2241  Post-ISR processed exposure.
2242  """
2243  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:711
def runDataRef(self, sensorRef)
Definition: isrTask.py:1329
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2003
def debugView(self, exposure, stepname)
Definition: isrTask.py:2144
def __init__(self, kwargs)
Definition: isrTask.py:701
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1422
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:824
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:778
def runDataRef(self, dataRef)
Definition: isrTask.py:2228
def __init__(self, args, kwargs)
Definition: isrTask.py:2224
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1966
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:761
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1868
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2067
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2190
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:748
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1570
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1466
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1819
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:821
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1381
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1757
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1800
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:952
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2090
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1502
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:771
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2114
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1708
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1893
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1928
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1844
def __init__(self, exposure, config)
Definition: isrTask.py:2174