lsst.ip.isr  17.0.1-15-g4fb9762
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
36 from lsst.afw.display import getDisplay
37 from lsst.afw.geom import Polygon
38 from lsst.daf.persistence import ButlerDataRef
39 from lsst.daf.persistence.butler import NoResults
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defects
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "RunIsrTask"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="Exposure",
78  dimensions=["instrument", "exposure", "detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["instrument", "calibration_label"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["instrument", "calibration_label", "detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["instrument", "calibration_label", "detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["instrument", "calibration_label"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="DefectsList",
120  dimensions=["instrument", "calibration_label", "detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["instrument", "physical_filter", "calibration_label"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["instrument", "calibration_label", "detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["instrument"],
149  )
150 
151  # output datasets
152  outputExposure = pipeBase.OutputDatasetField(
153  doc="Output ISR processed exposure.",
154  name="postISRCCD",
155  scalar=True,
156  storageClass="ExposureF",
157  dimensions=["instrument", "visit", "detector"],
158  )
159  outputOssThumbnail = pipeBase.OutputDatasetField(
160  doc="Output Overscan-subtracted thumbnail image.",
161  name="OssThumb",
162  scalar=True,
163  storageClass="Thumbnail",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167  doc="Output flat-corrected thumbnail image.",
168  name="FlattenedThumb",
169  scalar=True,
170  storageClass="TextStorage",
171  dimensions=["instrument", "visit", "detector"],
172  )
173 
174  quantum = pipeBase.QuantumConfig(
175  dimensions=["visit", "detector", "instrument"],
176  )
177 
178 
179  datasetType = pexConfig.Field(
180  dtype=str,
181  doc="Dataset type for input data; users will typically leave this alone, "
182  "but camera-specific ISR tasks will override it",
183  default="raw",
184  )
185 
186  fallbackFilterName = pexConfig.Field(
187  dtype=str,
188  doc="Fallback default filter name for calibrations.",
189  optional=True
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
195  )
196  fwhm = pexConfig.Field(
197  dtype=float,
198  doc="FWHM of PSF in arcseconds.",
199  default=1.0,
200  )
201  qa = pexConfig.ConfigField(
202  dtype=isrQa.IsrQaConfig,
203  doc="QA related configuration options.",
204  )
205 
206  # Image conversion configuration
207  doConvertIntToFloat = pexConfig.Field(
208  dtype=bool,
209  doc="Convert integer raw images to floating point values?",
210  default=True,
211  )
212 
213  # Saturated pixel handling.
214  doSaturation = pexConfig.Field(
215  dtype=bool,
216  doc="Mask saturated pixels? NB: this is totally independent of the"
217  " interpolation option - this is ONLY setting the bits in the mask."
218  " To have them interpolated make sure doSaturationInterpolation=True",
219  default=True,
220  )
221  saturatedMaskName = pexConfig.Field(
222  dtype=str,
223  doc="Name of mask plane to use in saturation detection and interpolation",
224  default="SAT",
225  )
226  saturation = pexConfig.Field(
227  dtype=float,
228  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
229  default=float("NaN"),
230  )
231  growSaturationFootprintSize = pexConfig.Field(
232  dtype=int,
233  doc="Number of pixels by which to grow the saturation footprints",
234  default=1,
235  )
236 
237  # Suspect pixel handling.
238  doSuspect = pexConfig.Field(
239  dtype=bool,
240  doc="Mask suspect pixels?",
241  default=True,
242  )
243  suspectMaskName = pexConfig.Field(
244  dtype=str,
245  doc="Name of mask plane to use for suspect pixels",
246  default="SUSPECT",
247  )
248  numEdgeSuspect = pexConfig.Field(
249  dtype=int,
250  doc="Number of edge pixels to be flagged as untrustworthy.",
251  default=0,
252  )
253 
254  # Initial masking options.
255  doSetBadRegions = pexConfig.Field(
256  dtype=bool,
257  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
258  default=True,
259  )
260  badStatistic = pexConfig.ChoiceField(
261  dtype=str,
262  doc="How to estimate the average value for BAD regions.",
263  default='MEANCLIP',
264  allowed={
265  "MEANCLIP": "Correct using the (clipped) mean of good data",
266  "MEDIAN": "Correct using the median of the good data",
267  },
268  )
269 
270  # Overscan subtraction configuration.
271  doOverscan = pexConfig.Field(
272  dtype=bool,
273  doc="Do overscan subtraction?",
274  default=True,
275  )
276  overscanFitType = pexConfig.ChoiceField(
277  dtype=str,
278  doc="The method for fitting the overscan bias level.",
279  default='MEDIAN',
280  allowed={
281  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
282  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
283  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
284  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
285  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
286  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
287  "MEAN": "Correct using the mean of the overscan region",
288  "MEANCLIP": "Correct using a clipped mean of the overscan region",
289  "MEDIAN": "Correct using the median of the overscan region",
290  },
291  )
292  overscanOrder = pexConfig.Field(
293  dtype=int,
294  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
295  "or number of spline knots if overscan fit type is a spline."),
296  default=1,
297  )
298  overscanNumSigmaClip = pexConfig.Field(
299  dtype=float,
300  doc="Rejection threshold (sigma) for collapsing overscan before fit",
301  default=3.0,
302  )
303  overscanIsInt = pexConfig.Field(
304  dtype=bool,
305  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
306  default=True,
307  )
308  overscanNumLeadingColumnsToSkip = pexConfig.Field(
309  dtype=int,
310  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
311  default=0,
312  )
313  overscanNumTrailingColumnsToSkip = pexConfig.Field(
314  dtype=int,
315  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
316  default=0,
317  )
318  overscanMaxDev = pexConfig.Field(
319  dtype=float,
320  doc="Maximum deviation from the median for overscan",
321  default=1000.0, check=lambda x: x > 0
322  )
323  overscanBiasJump = pexConfig.Field(
324  dtype=bool,
325  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
326  default=False,
327  )
328  overscanBiasJumpKeyword = pexConfig.Field(
329  dtype=str,
330  doc="Header keyword containing information about devices.",
331  default="NO_SUCH_KEY",
332  )
333  overscanBiasJumpDevices = pexConfig.ListField(
334  dtype=str,
335  doc="List of devices that need piecewise overscan correction.",
336  default=(),
337  )
338  overscanBiasJumpLocation = pexConfig.Field(
339  dtype=int,
340  doc="Location of bias jump along y-axis.",
341  default=0,
342  )
343 
344  # Amplifier to CCD assembly configuration
345  doAssembleCcd = pexConfig.Field(
346  dtype=bool,
347  default=True,
348  doc="Assemble amp-level exposures into a ccd-level exposure?"
349  )
350  assembleCcd = pexConfig.ConfigurableField(
351  target=AssembleCcdTask,
352  doc="CCD assembly task",
353  )
354 
355  # General calibration configuration.
356  doAssembleIsrExposures = pexConfig.Field(
357  dtype=bool,
358  default=False,
359  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
360  )
361  doTrimToMatchCalib = pexConfig.Field(
362  dtype=bool,
363  default=False,
364  doc="Trim raw data to match calibration bounding boxes?"
365  )
366 
367  # Bias subtraction.
368  doBias = pexConfig.Field(
369  dtype=bool,
370  doc="Apply bias frame correction?",
371  default=True,
372  )
373  biasDataProductName = pexConfig.Field(
374  dtype=str,
375  doc="Name of the bias data product",
376  default="bias",
377  )
378 
379  # Variance construction
380  doVariance = pexConfig.Field(
381  dtype=bool,
382  doc="Calculate variance?",
383  default=True
384  )
385  gain = pexConfig.Field(
386  dtype=float,
387  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
388  default=float("NaN"),
389  )
390  readNoise = pexConfig.Field(
391  dtype=float,
392  doc="The read noise to use if no Detector is present in the Exposure",
393  default=0.0,
394  )
395  doEmpiricalReadNoise = pexConfig.Field(
396  dtype=bool,
397  default=False,
398  doc="Calculate empirical read noise instead of value from AmpInfo data?"
399  )
400 
401  # Linearization.
402  doLinearize = pexConfig.Field(
403  dtype=bool,
404  doc="Correct for nonlinearity of the detector's response?",
405  default=True,
406  )
407 
408  # Crosstalk.
409  doCrosstalk = pexConfig.Field(
410  dtype=bool,
411  doc="Apply intra-CCD crosstalk correction?",
412  default=False,
413  )
414  doCrosstalkBeforeAssemble = pexConfig.Field(
415  dtype=bool,
416  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
417  default=True,
418  )
419  crosstalk = pexConfig.ConfigurableField(
420  target=CrosstalkTask,
421  doc="Intra-CCD crosstalk correction",
422  )
423 
424  # Masking option prior to brighter-fatter.
425  doWidenSaturationTrails = pexConfig.Field(
426  dtype=bool,
427  doc="Widen bleed trails based on their width?",
428  default=True
429  )
430 
431  # Brighter-Fatter correction.
432  doBrighterFatter = pexConfig.Field(
433  dtype=bool,
434  default=False,
435  doc="Apply the brighter fatter correction"
436  )
437  brighterFatterLevel = pexConfig.ChoiceField(
438  dtype=str,
439  default="DETECTOR",
440  doc="The level at which to correct for brighter-fatter.",
441  allowed={
442  "AMP": "Every amplifier treated separately.",
443  "DETECTOR": "One kernel per detector",
444  }
445  )
446  brighterFatterKernelFile = pexConfig.Field(
447  dtype=str,
448  default='',
449  doc="Kernel file used for the brighter fatter correction"
450  )
451  brighterFatterMaxIter = pexConfig.Field(
452  dtype=int,
453  default=10,
454  doc="Maximum number of iterations for the brighter fatter correction"
455  )
456  brighterFatterThreshold = pexConfig.Field(
457  dtype=float,
458  default=1000,
459  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
460  " absolute value of the difference between the current corrected image and the one"
461  " from the previous iteration summed over all the pixels."
462  )
463  brighterFatterApplyGain = pexConfig.Field(
464  dtype=bool,
465  default=True,
466  doc="Should the gain be applied when applying the brighter fatter correction?"
467  )
468 
469  # Defect and bad pixel correction options.
470  doDefect = pexConfig.Field(
471  dtype=bool,
472  doc="Apply correction for CCD defects, e.g. hot pixels?",
473  default=True,
474  )
475  doSaturationInterpolation = pexConfig.Field(
476  dtype=bool,
477  doc="Perform interpolation over pixels masked as saturated?"
478  " NB: This is independent of doSaturation; if that is False this plane"
479  " will likely be blank, resulting in a no-op here.",
480  default=True,
481  )
482  numEdgeSuspect = pexConfig.Field(
483  dtype=int,
484  doc="Number of edge pixels to be flagged as untrustworthy.",
485  default=0,
486  )
487 
488  # Dark subtraction.
489  doDark = pexConfig.Field(
490  dtype=bool,
491  doc="Apply dark frame correction?",
492  default=True,
493  )
494  darkDataProductName = pexConfig.Field(
495  dtype=str,
496  doc="Name of the dark data product",
497  default="dark",
498  )
499 
500  # Camera-specific stray light removal.
501  doStrayLight = pexConfig.Field(
502  dtype=bool,
503  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
504  default=False,
505  )
506  strayLight = pexConfig.ConfigurableField(
507  target=StrayLightTask,
508  doc="y-band stray light correction"
509  )
510 
511  # Flat correction.
512  doFlat = pexConfig.Field(
513  dtype=bool,
514  doc="Apply flat field correction?",
515  default=True,
516  )
517  flatDataProductName = pexConfig.Field(
518  dtype=str,
519  doc="Name of the flat data product",
520  default="flat",
521  )
522  flatScalingType = pexConfig.ChoiceField(
523  dtype=str,
524  doc="The method for scaling the flat on the fly.",
525  default='USER',
526  allowed={
527  "USER": "Scale by flatUserScale",
528  "MEAN": "Scale by the inverse of the mean",
529  "MEDIAN": "Scale by the inverse of the median",
530  },
531  )
532  flatUserScale = pexConfig.Field(
533  dtype=float,
534  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
535  default=1.0,
536  )
537  doTweakFlat = pexConfig.Field(
538  dtype=bool,
539  doc="Tweak flats to match observed amplifier ratios?",
540  default=False
541  )
542 
543  # Amplifier normalization based on gains instead of using flats configuration.
544  doApplyGains = pexConfig.Field(
545  dtype=bool,
546  doc="Correct the amplifiers for their gains instead of applying flat correction",
547  default=False,
548  )
549  normalizeGains = pexConfig.Field(
550  dtype=bool,
551  doc="Normalize all the amplifiers in each CCD to have the same median value.",
552  default=False,
553  )
554 
555  # Fringe correction.
556  doFringe = pexConfig.Field(
557  dtype=bool,
558  doc="Apply fringe correction?",
559  default=True,
560  )
561  fringe = pexConfig.ConfigurableField(
562  target=FringeTask,
563  doc="Fringe subtraction task",
564  )
565  fringeAfterFlat = pexConfig.Field(
566  dtype=bool,
567  doc="Do fringe subtraction after flat-fielding?",
568  default=True,
569  )
570 
571  # NAN pixel interpolation option.
572  doNanInterpAfterFlat = pexConfig.Field(
573  dtype=bool,
574  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
575  "also have to interpolate them before flat-fielding."),
576  default=False,
577  )
578 
579  # Distortion model application.
580  doAddDistortionModel = pexConfig.Field(
581  dtype=bool,
582  doc="Apply a distortion model based on camera geometry to the WCS?",
583  default=True,
584  )
585 
586  # Initial CCD-level background statistics options.
587  doMeasureBackground = pexConfig.Field(
588  dtype=bool,
589  doc="Measure the background level on the reduced image?",
590  default=False,
591  )
592 
593  # Camera-specific masking configuration.
594  doCameraSpecificMasking = pexConfig.Field(
595  dtype=bool,
596  doc="Mask camera-specific bad regions?",
597  default=False,
598  )
599  masking = pexConfig.ConfigurableField(
600  target=MaskingTask,
601  doc="Masking task."
602  )
603 
604  # Default photometric calibration options.
605  fluxMag0T1 = pexConfig.DictField(
606  keytype=str,
607  itemtype=float,
608  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
609  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
610  ))
611  )
612  defaultFluxMag0T1 = pexConfig.Field(
613  dtype=float,
614  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
615  default=pow(10.0, 0.4*28.0)
616  )
617 
618  # Vignette correction configuration.
619  doVignette = pexConfig.Field(
620  dtype=bool,
621  doc="Apply vignetting parameters?",
622  default=False,
623  )
624  vignette = pexConfig.ConfigurableField(
625  target=VignetteTask,
626  doc="Vignetting task.",
627  )
628 
629  # Transmission curve configuration.
630  doAttachTransmissionCurve = pexConfig.Field(
631  dtype=bool,
632  default=False,
633  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
634  )
635  doUseOpticsTransmission = pexConfig.Field(
636  dtype=bool,
637  default=True,
638  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
639  )
640  doUseFilterTransmission = pexConfig.Field(
641  dtype=bool,
642  default=True,
643  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
644  )
645  doUseSensorTransmission = pexConfig.Field(
646  dtype=bool,
647  default=True,
648  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
649  )
650  doUseAtmosphereTransmission = pexConfig.Field(
651  dtype=bool,
652  default=True,
653  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
654  )
655 
656  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
657  doWrite = pexConfig.Field(
658  dtype=bool,
659  doc="Persist postISRCCD?",
660  default=True,
661  )
662 
663  def validate(self):
664  super().validate()
665  if self.doFlat and self.doApplyGains:
666  raise ValueError("You may not specify both doFlat and doApplyGains")
667 
668 
669 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
670  """Apply common instrument signature correction algorithms to a raw frame.
671 
672  The process for correcting imaging data is very similar from
673  camera to camera. This task provides a vanilla implementation of
674  doing these corrections, including the ability to turn certain
675  corrections off if they are not needed. The inputs to the primary
676  method, `run()`, are a raw exposure to be corrected and the
677  calibration data products. The raw input is a single chip sized
678  mosaic of all amps including overscans and other non-science
679  pixels. The method `runDataRef()` identifies and defines the
680  calibration data products, and is intended for use by a
681  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
682  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
683  subclassed for different camera, although the most camera specific
684  methods have been split into subtasks that can be redirected
685  appropriately.
686 
687  The __init__ method sets up the subtasks for ISR processing, using
688  the defaults from `lsst.ip.isr`.
689 
690  Parameters
691  ----------
692  args : `list`
693  Positional arguments passed to the Task constructor. None used at this time.
694  kwargs : `dict`, optional
695  Keyword arguments passed on to the Task constructor. None used at this time.
696  """
697  ConfigClass = IsrTaskConfig
698  _DefaultName = "isr"
699 
700  def __init__(self, **kwargs):
701  super().__init__(**kwargs)
702  self.makeSubtask("assembleCcd")
703  self.makeSubtask("crosstalk")
704  self.makeSubtask("strayLight")
705  self.makeSubtask("fringe")
706  self.makeSubtask("masking")
707  self.makeSubtask("vignette")
708 
709  @classmethod
710  def getInputDatasetTypes(cls, config):
711  inputTypeDict = super().getInputDatasetTypes(config)
712 
713  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
714  # need because the configuration tells us we will not be bothering with the
715  # correction that uses that IDT.
716  if config.doBias is not True:
717  inputTypeDict.pop("bias", None)
718  if config.doLinearize is not True:
719  inputTypeDict.pop("linearizer", None)
720  if config.doCrosstalk is not True:
721  inputTypeDict.pop("crosstalkSources", None)
722  if config.doBrighterFatter is not True:
723  inputTypeDict.pop("bfKernel", None)
724  if config.doDefect is not True:
725  inputTypeDict.pop("defects", None)
726  if config.doDark is not True:
727  inputTypeDict.pop("dark", None)
728  if config.doFlat is not True:
729  inputTypeDict.pop("flat", None)
730  if config.doAttachTransmissionCurve is not True:
731  inputTypeDict.pop("opticsTransmission", None)
732  inputTypeDict.pop("filterTransmission", None)
733  inputTypeDict.pop("sensorTransmission", None)
734  inputTypeDict.pop("atmosphereTransmission", None)
735  if config.doUseOpticsTransmission is not True:
736  inputTypeDict.pop("opticsTransmission", None)
737  if config.doUseFilterTransmission is not True:
738  inputTypeDict.pop("filterTransmission", None)
739  if config.doUseSensorTransmission is not True:
740  inputTypeDict.pop("sensorTransmission", None)
741  if config.doUseAtmosphereTransmission is not True:
742  inputTypeDict.pop("atmosphereTransmission", None)
743 
744  return inputTypeDict
745 
746  @classmethod
747  def getOutputDatasetTypes(cls, config):
748  outputTypeDict = super().getOutputDatasetTypes(config)
749 
750  if config.qa.doThumbnailOss is not True:
751  outputTypeDict.pop("outputOssThumbnail", None)
752  if config.qa.doThumbnailFlattened is not True:
753  outputTypeDict.pop("outputFlattenedThumbnail", None)
754  if config.doWrite is not True:
755  outputTypeDict.pop("outputExposure", None)
756 
757  return outputTypeDict
758 
759  @classmethod
760  def getPrerequisiteDatasetTypes(cls, config):
761  # Input calibration datasets should not constrain the QuantumGraph
762  # (it'd be confusing if not having flats just silently resulted in no
763  # data being processed). Our nomenclature for that is that these are
764  # "prerequisite" datasets (only "ccdExposure" == "raw" isn't).
765  names = set(cls.getInputDatasetTypes(config))
766  names.remove("ccdExposure")
767  return names
768 
769  @classmethod
770  def getPerDatasetTypeDimensions(cls, config):
771  # Input calibration datasets of different types (i.e. flat, bias) need
772  # not have the same validity range. That makes calibration_label
773  # (which maps directly to a validity range) a "per-DatasetType
774  # dimension".
775  return frozenset(["calibration_label"])
776 
777  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
778  try:
779  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
780  except Exception as e:
781  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
782 
783  inputData['isGen3'] = True
784 
785  if self.config.doLinearize is True:
786  if 'linearizer' not in inputData.keys():
787  detector = inputData['camera'][inputData['detectorNum']]
788  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
789  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
790 
791  if inputData['defects'] is not None:
792  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
793  # masking expects a list of defects defined by their bounding box
794  if not isinstance(inputData["defects"], Defects):
795  inputData["defects"] = Defects.fromTable(inputData["defects"])
796 
797  # Broken: DM-17169
798  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
799  # fixed for non-HSC cameras in the future.
800  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
801  # if self.config.doCrosstalk else None)
802 
803  # Broken: DM-17152
804  # Fringes are not tested to be handled correctly by Gen3 butler.
805  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
806  # assembler=self.assembleCcd
807  # if self.config.doAssembleIsrExposures else None)
808  # if self.config.doFringe and
809  # self.fringe.checkFilter(inputData['ccdExposure'])
810  # else pipeBase.Struct(fringes=None))
811 
812  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
813 
814  def makeDatasetType(self, dsConfig):
815  return super().makeDatasetType(dsConfig)
816 
817  def readIsrData(self, dataRef, rawExposure):
818  """!Retrieve necessary frames for instrument signature removal.
819 
820  Pre-fetching all required ISR data products limits the IO
821  required by the ISR. Any conflict between the calibration data
822  available and that needed for ISR is also detected prior to
823  doing processing, allowing it to fail quickly.
824 
825  Parameters
826  ----------
827  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
828  Butler reference of the detector data to be processed
829  rawExposure : `afw.image.Exposure`
830  The raw exposure that will later be corrected with the
831  retrieved calibration data; should not be modified in this
832  method.
833 
834  Returns
835  -------
836  result : `lsst.pipe.base.Struct`
837  Result struct with components (which may be `None`):
838  - ``bias``: bias calibration frame (`afw.image.Exposure`)
839  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
840  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
841  - ``dark``: dark calibration frame (`afw.image.Exposure`)
842  - ``flat``: flat calibration frame (`afw.image.Exposure`)
843  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
844  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
845  - ``fringes``: `lsst.pipe.base.Struct` with components:
846  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
847  - ``seed``: random seed derived from the ccdExposureId for random
848  number generator (`uint32`)
849  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
850  A ``TransmissionCurve`` that represents the throughput of the optics,
851  to be evaluated in focal-plane coordinates.
852  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
853  A ``TransmissionCurve`` that represents the throughput of the filter
854  itself, to be evaluated in focal-plane coordinates.
855  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
856  A ``TransmissionCurve`` that represents the throughput of the sensor
857  itself, to be evaluated in post-assembly trimmed detector coordinates.
858  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
859  A ``TransmissionCurve`` that represents the throughput of the
860  atmosphere, assumed to be spatially constant.
861  - ``strayLightData`` : `object`
862  An opaque object containing calibration information for
863  stray-light correction. If `None`, no correction will be
864  performed.
865 
866  Raises
867  ------
868  NotImplementedError :
869  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
870  """
871  ccd = rawExposure.getDetector()
872  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
873  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
874  if self.config.doBias else None)
875  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
876  linearizer = (dataRef.get("linearizer", immediate=True)
877  if self.doLinearize(ccd) else None)
878  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
879  if self.config.doCrosstalk else None)
880  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
881  if self.config.doDark else None)
882  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
883  if self.config.doFlat else None)
884 
885  brighterFatterKernel = None
886  if self.config.doBrighterFatter is True:
887 
888  # Use the new-style cp_pipe version of the kernel is it exists.
889  try:
890  brighterFatterKernel = dataRef.get("brighterFatterKernel")
891  except NoResults:
892  # Fall back to the old-style numpy-ndarray style kernel if necessary.
893  try:
894  brighterFatterKernel = dataRef.get("bfKernel")
895  except NoResults:
896  brighterFatterKernel = None
897  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
898  # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
899  # this detector, or raise an error.
900  if self.config.brighterFatterLevel == 'DETECTOR':
901  brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
902  else:
903  # TODO DM-15631 for implementing this
904  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
905 
906  defectList = (dataRef.get("defects")
907  if self.config.doDefect else None)
908  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
909  if self.config.doAssembleIsrExposures else None)
910  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
911  else pipeBase.Struct(fringes=None))
912 
913  if self.config.doAttachTransmissionCurve:
914  opticsTransmission = (dataRef.get("transmission_optics")
915  if self.config.doUseOpticsTransmission else None)
916  filterTransmission = (dataRef.get("transmission_filter")
917  if self.config.doUseFilterTransmission else None)
918  sensorTransmission = (dataRef.get("transmission_sensor")
919  if self.config.doUseSensorTransmission else None)
920  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
921  if self.config.doUseAtmosphereTransmission else None)
922  else:
923  opticsTransmission = None
924  filterTransmission = None
925  sensorTransmission = None
926  atmosphereTransmission = None
927 
928  if self.config.doStrayLight:
929  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
930  else:
931  strayLightData = None
932 
933  # Struct should include only kwargs to run()
934  return pipeBase.Struct(bias=biasExposure,
935  linearizer=linearizer,
936  crosstalkSources=crosstalkSources,
937  dark=darkExposure,
938  flat=flatExposure,
939  bfKernel=brighterFatterKernel,
940  defects=defectList,
941  fringes=fringeStruct,
942  opticsTransmission=opticsTransmission,
943  filterTransmission=filterTransmission,
944  sensorTransmission=sensorTransmission,
945  atmosphereTransmission=atmosphereTransmission,
946  strayLightData=strayLightData
947  )
948 
949  @pipeBase.timeMethod
950  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
951  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
952  opticsTransmission=None, filterTransmission=None,
953  sensorTransmission=None, atmosphereTransmission=None,
954  detectorNum=None, strayLightData=None, isGen3=False,
955  ):
956  """!Perform instrument signature removal on an exposure.
957 
958  Steps included in the ISR processing, in order performed, are:
959  - saturation and suspect pixel masking
960  - overscan subtraction
961  - CCD assembly of individual amplifiers
962  - bias subtraction
963  - variance image construction
964  - linearization of non-linear response
965  - crosstalk masking
966  - brighter-fatter correction
967  - dark subtraction
968  - fringe correction
969  - stray light subtraction
970  - flat correction
971  - masking of known defects and camera specific features
972  - vignette calculation
973  - appending transmission curve and distortion model
974 
975  Parameters
976  ----------
977  ccdExposure : `lsst.afw.image.Exposure`
978  The raw exposure that is to be run through ISR. The
979  exposure is modified by this method.
980  camera : `lsst.afw.cameraGeom.Camera`, optional
981  The camera geometry for this exposure. Used to select the
982  distortion model appropriate for this data.
983  bias : `lsst.afw.image.Exposure`, optional
984  Bias calibration frame.
985  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
986  Functor for linearization.
987  crosstalkSources : `list`, optional
988  List of possible crosstalk sources.
989  dark : `lsst.afw.image.Exposure`, optional
990  Dark calibration frame.
991  flat : `lsst.afw.image.Exposure`, optional
992  Flat calibration frame.
993  bfKernel : `numpy.ndarray`, optional
994  Brighter-fatter kernel.
995  defects : `lsst.meas.algorithms.Defects`, optional
996  List of defects.
997  fringes : `lsst.pipe.base.Struct`, optional
998  Struct containing the fringe correction data, with
999  elements:
1000  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1001  - ``seed``: random seed derived from the ccdExposureId for random
1002  number generator (`uint32`)
1003  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1004  A ``TransmissionCurve`` that represents the throughput of the optics,
1005  to be evaluated in focal-plane coordinates.
1006  filterTransmission : `lsst.afw.image.TransmissionCurve`
1007  A ``TransmissionCurve`` that represents the throughput of the filter
1008  itself, to be evaluated in focal-plane coordinates.
1009  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1010  A ``TransmissionCurve`` that represents the throughput of the sensor
1011  itself, to be evaluated in post-assembly trimmed detector coordinates.
1012  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1013  A ``TransmissionCurve`` that represents the throughput of the
1014  atmosphere, assumed to be spatially constant.
1015  detectorNum : `int`, optional
1016  The integer number for the detector to process.
1017  isGen3 : bool, optional
1018  Flag this call to run() as using the Gen3 butler environment.
1019  strayLightData : `object`, optional
1020  Opaque object containing calibration information for stray-light
1021  correction. If `None`, no correction will be performed.
1022 
1023  Returns
1024  -------
1025  result : `lsst.pipe.base.Struct`
1026  Result struct with component:
1027  - ``exposure`` : `afw.image.Exposure`
1028  The fully ISR corrected exposure.
1029  - ``outputExposure`` : `afw.image.Exposure`
1030  An alias for `exposure`
1031  - ``ossThumb`` : `numpy.ndarray`
1032  Thumbnail image of the exposure after overscan subtraction.
1033  - ``flattenedThumb`` : `numpy.ndarray`
1034  Thumbnail image of the exposure after flat-field correction.
1035 
1036  Raises
1037  ------
1038  RuntimeError
1039  Raised if a configuration option is set to True, but the
1040  required calibration data has not been specified.
1041 
1042  Notes
1043  -----
1044  The current processed exposure can be viewed by setting the
1045  appropriate lsstDebug entries in the `debug.display`
1046  dictionary. The names of these entries correspond to some of
1047  the IsrTaskConfig Boolean options, with the value denoting the
1048  frame to use. The exposure is shown inside the matching
1049  option check and after the processing of that step has
1050  finished. The steps with debug points are:
1051 
1052  doAssembleCcd
1053  doBias
1054  doCrosstalk
1055  doBrighterFatter
1056  doDark
1057  doFringe
1058  doStrayLight
1059  doFlat
1060 
1061  In addition, setting the "postISRCCD" entry displays the
1062  exposure after all ISR processing has finished.
1063 
1064  """
1065 
1066  if isGen3 is True:
1067  # Gen3 currently cannot automatically do configuration overrides.
1068  # DM-15257 looks to discuss this issue.
1069 
1070  self.config.doFringe = False
1071 
1072  # Configure input exposures;
1073  if detectorNum is None:
1074  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1075 
1076  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1077  bias = self.ensureExposure(bias, camera, detectorNum)
1078  dark = self.ensureExposure(dark, camera, detectorNum)
1079  flat = self.ensureExposure(flat, camera, detectorNum)
1080  else:
1081  if isinstance(ccdExposure, ButlerDataRef):
1082  return self.runDataRef(ccdExposure)
1083 
1084  ccd = ccdExposure.getDetector()
1085 
1086  if not ccd:
1087  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1088  ccd = [FakeAmp(ccdExposure, self.config)]
1089 
1090  # Validate Input
1091  if self.config.doBias and bias is None:
1092  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1093  if self.doLinearize(ccd) and linearizer is None:
1094  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1095  if self.config.doBrighterFatter and bfKernel is None:
1096  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1097  if self.config.doDark and dark is None:
1098  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1099  if fringes is None:
1100  fringes = pipeBase.Struct(fringes=None)
1101  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1102  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1103  if self.config.doFlat and flat is None:
1104  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1105  if self.config.doDefect and defects is None:
1106  raise RuntimeError("Must supply defects if config.doDefect=True.")
1107  if self.config.doAddDistortionModel and camera is None:
1108  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1109 
1110  # Begin ISR processing.
1111  if self.config.doConvertIntToFloat:
1112  self.log.info("Converting exposure to floating point values")
1113  ccdExposure = self.convertIntToFloat(ccdExposure)
1114 
1115  # Amplifier level processing.
1116  overscans = []
1117  for amp in ccd:
1118  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1119  if ccdExposure.getBBox().contains(amp.getBBox()):
1120  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1121  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1122 
1123  if self.config.doOverscan and not badAmp:
1124  # Overscan correction on amp-by-amp basis.
1125  overscanResults = self.overscanCorrection(ccdExposure, amp)
1126  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1127  if self.config.qa is not None and self.config.qa.saveStats is True:
1128  if isinstance(overscanResults.overscanFit, float):
1129  qaMedian = overscanResults.overscanFit
1130  qaStdev = float("NaN")
1131  else:
1132  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1133  afwMath.MEDIAN | afwMath.STDEVCLIP)
1134  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1135  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1136 
1137  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1138  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1139  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1140  (amp.getName(), qaMedian, qaStdev))
1141  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1142  else:
1143  if badAmp:
1144  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1145  overscanResults = None
1146 
1147  overscans.append(overscanResults if overscanResults is not None else None)
1148  else:
1149  self.log.info("Skipped OSCAN")
1150 
1151  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1152  self.log.info("Applying crosstalk correction.")
1153  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1154  self.debugView(ccdExposure, "doCrosstalk")
1155 
1156  if self.config.doAssembleCcd:
1157  self.log.info("Assembling CCD from amplifiers")
1158  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1159 
1160  if self.config.expectWcs and not ccdExposure.getWcs():
1161  self.log.warn("No WCS found in input exposure")
1162  self.debugView(ccdExposure, "doAssembleCcd")
1163 
1164  ossThumb = None
1165  if self.config.qa.doThumbnailOss:
1166  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1167 
1168  if self.config.doBias:
1169  self.log.info("Applying bias correction.")
1170  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1171  trimToFit=self.config.doTrimToMatchCalib)
1172  self.debugView(ccdExposure, "doBias")
1173 
1174  if self.config.doVariance:
1175  for amp, overscanResults in zip(ccd, overscans):
1176  if ccdExposure.getBBox().contains(amp.getBBox()):
1177  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1178  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1179  if overscanResults is not None:
1180  self.updateVariance(ampExposure, amp,
1181  overscanImage=overscanResults.overscanImage)
1182  else:
1183  self.updateVariance(ampExposure, amp,
1184  overscanImage=None)
1185  if self.config.qa is not None and self.config.qa.saveStats is True:
1186  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1187  afwMath.MEDIAN | afwMath.STDEVCLIP)
1188  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1189  qaStats.getValue(afwMath.MEDIAN))
1190  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1191  qaStats.getValue(afwMath.STDEVCLIP))
1192  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1193  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1194  qaStats.getValue(afwMath.STDEVCLIP)))
1195 
1196  if self.doLinearize(ccd):
1197  self.log.info("Applying linearizer.")
1198  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1199 
1200  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1201  self.log.info("Applying crosstalk correction.")
1202  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1203  self.debugView(ccdExposure, "doCrosstalk")
1204 
1205  if self.config.doWidenSaturationTrails:
1206  self.log.info("Widening saturation trails.")
1207  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1208 
1209  interpolationDone = False
1210  if self.config.doBrighterFatter:
1211  # We need to apply flats and darks before we can interpolate, and we
1212  # need to interpolate before we do B-F, but we do B-F without the
1213  # flats and darks applied so we can work in units of electrons or holes.
1214  # This context manager applies and then removes the darks and flats.
1215  with self.flatContext(ccdExposure, flat, dark):
1216  if self.config.doDefect:
1217  self.maskAndInterpDefect(ccdExposure, defects)
1218 
1219  if self.config.doSaturationInterpolation:
1220  self.saturationInterpolation(ccdExposure)
1221 
1222  self.maskAndInterpNan(ccdExposure)
1223  interpolationDone = True
1224 
1225  self.log.info("Applying brighter fatter correction.")
1226  isrFunctions.brighterFatterCorrection(ccdExposure, bfKernel,
1227  self.config.brighterFatterMaxIter,
1228  self.config.brighterFatterThreshold,
1229  self.config.brighterFatterApplyGain,
1230  )
1231  self.debugView(ccdExposure, "doBrighterFatter")
1232 
1233  if self.config.doDark:
1234  self.log.info("Applying dark correction.")
1235  self.darkCorrection(ccdExposure, dark)
1236  self.debugView(ccdExposure, "doDark")
1237 
1238  if self.config.doFringe and not self.config.fringeAfterFlat:
1239  self.log.info("Applying fringe correction before flat.")
1240  self.fringe.run(ccdExposure, **fringes.getDict())
1241  self.debugView(ccdExposure, "doFringe")
1242 
1243  if self.config.doStrayLight:
1244  if strayLightData is not None:
1245  self.log.info("Applying stray light correction.")
1246  self.strayLight.run(ccdExposure, strayLightData)
1247  self.debugView(ccdExposure, "doStrayLight")
1248  else:
1249  self.log.debug("Skipping stray light correction: no data found for this image.")
1250 
1251  if self.config.doFlat:
1252  self.log.info("Applying flat correction.")
1253  self.flatCorrection(ccdExposure, flat)
1254  self.debugView(ccdExposure, "doFlat")
1255 
1256  if self.config.doApplyGains:
1257  self.log.info("Applying gain correction instead of flat.")
1258  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1259 
1260  if self.config.doDefect and not interpolationDone:
1261  self.log.info("Masking and interpolating defects.")
1262  self.maskAndInterpDefect(ccdExposure, defects)
1263 
1264  if self.config.doSaturationInterpolation and not interpolationDone:
1265  self.log.info("Interpolating saturated pixels.")
1266  self.saturationInterpolation(ccdExposure)
1267 
1268  if self.config.doNanInterpAfterFlat or not interpolationDone:
1269  self.log.info("Masking and interpolating NAN value pixels.")
1270  self.maskAndInterpNan(ccdExposure)
1271 
1272  if self.config.doFringe and self.config.fringeAfterFlat:
1273  self.log.info("Applying fringe correction after flat.")
1274  self.fringe.run(ccdExposure, **fringes.getDict())
1275 
1276  if self.config.doSetBadRegions:
1277  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1278  if badPixelCount > 0:
1279  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1280 
1281  flattenedThumb = None
1282  if self.config.qa.doThumbnailFlattened:
1283  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1284 
1285  if self.config.doCameraSpecificMasking:
1286  self.log.info("Masking regions for camera specific reasons.")
1287  self.masking.run(ccdExposure)
1288 
1289  self.roughZeroPoint(ccdExposure)
1290 
1291  if self.config.doVignette:
1292  self.log.info("Constructing Vignette polygon.")
1293  self.vignettePolygon = self.vignette.run(ccdExposure)
1294 
1295  if self.config.vignette.doWriteVignettePolygon:
1296  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1297 
1298  if self.config.doAttachTransmissionCurve:
1299  self.log.info("Adding transmission curves.")
1300  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1301  filterTransmission=filterTransmission,
1302  sensorTransmission=sensorTransmission,
1303  atmosphereTransmission=atmosphereTransmission)
1304 
1305  if self.config.doAddDistortionModel:
1306  self.log.info("Adding a distortion model to the WCS.")
1307  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1308 
1309  if self.config.doMeasureBackground:
1310  self.log.info("Measuring background level:")
1311  self.measureBackground(ccdExposure, self.config.qa)
1312 
1313  if self.config.qa is not None and self.config.qa.saveStats is True:
1314  for amp in ccd:
1315  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1316  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1317  afwMath.MEDIAN | afwMath.STDEVCLIP)
1318  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1319  qaStats.getValue(afwMath.MEDIAN))
1320  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1321  qaStats.getValue(afwMath.STDEVCLIP))
1322  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1323  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1324  qaStats.getValue(afwMath.STDEVCLIP)))
1325 
1326  self.debugView(ccdExposure, "postISRCCD")
1327 
1328  return pipeBase.Struct(
1329  exposure=ccdExposure,
1330  ossThumb=ossThumb,
1331  flattenedThumb=flattenedThumb,
1332 
1333  outputExposure=ccdExposure,
1334  outputOssThumbnail=ossThumb,
1335  outputFlattenedThumbnail=flattenedThumb,
1336  )
1337 
1338  @pipeBase.timeMethod
1339  def runDataRef(self, sensorRef):
1340  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1341 
1342  This method contains the `CmdLineTask` interface to the ISR
1343  processing. All IO is handled here, freeing the `run()` method
1344  to manage only pixel-level calculations. The steps performed
1345  are:
1346  - Read in necessary detrending/isr/calibration data.
1347  - Process raw exposure in `run()`.
1348  - Persist the ISR-corrected exposure as "postISRCCD" if
1349  config.doWrite=True.
1350 
1351  Parameters
1352  ----------
1353  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1354  DataRef of the detector data to be processed
1355 
1356  Returns
1357  -------
1358  result : `lsst.pipe.base.Struct`
1359  Result struct with component:
1360  - ``exposure`` : `afw.image.Exposure`
1361  The fully ISR corrected exposure.
1362 
1363  Raises
1364  ------
1365  RuntimeError
1366  Raised if a configuration option is set to True, but the
1367  required calibration data does not exist.
1368 
1369  """
1370  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1371 
1372  ccdExposure = sensorRef.get(self.config.datasetType)
1373 
1374  camera = sensorRef.get("camera")
1375  if camera is None and self.config.doAddDistortionModel:
1376  raise RuntimeError("config.doAddDistortionModel is True "
1377  "but could not get a camera from the butler")
1378  isrData = self.readIsrData(sensorRef, ccdExposure)
1379 
1380  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1381 
1382  if self.config.doWrite:
1383  sensorRef.put(result.exposure, "postISRCCD")
1384  if result.ossThumb is not None:
1385  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1386  if result.flattenedThumb is not None:
1387  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1388 
1389  return result
1390 
1391  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1392  """!Retrieve a calibration dataset for removing instrument signature.
1393 
1394  Parameters
1395  ----------
1396 
1397  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1398  DataRef of the detector data to find calibration datasets
1399  for.
1400  datasetType : `str`
1401  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1402  immediate : `Bool`
1403  If True, disable butler proxies to enable error handling
1404  within this routine.
1405 
1406  Returns
1407  -------
1408  exposure : `lsst.afw.image.Exposure`
1409  Requested calibration frame.
1410 
1411  Raises
1412  ------
1413  RuntimeError
1414  Raised if no matching calibration frame can be found.
1415  """
1416  try:
1417  exp = dataRef.get(datasetType, immediate=immediate)
1418  except Exception as exc1:
1419  if not self.config.fallbackFilterName:
1420  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1421  try:
1422  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1423  except Exception as exc2:
1424  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1425  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1426  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1427 
1428  if self.config.doAssembleIsrExposures:
1429  exp = self.assembleCcd.assembleCcd(exp)
1430  return exp
1431 
1432  def ensureExposure(self, inputExp, camera, detectorNum):
1433  """Ensure that the data returned by Butler is a fully constructed exposure.
1434 
1435  ISR requires exposure-level image data for historical reasons, so if we did
1436  not recieve that from Butler, construct it from what we have, modifying the
1437  input in place.
1438 
1439  Parameters
1440  ----------
1441  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1442  `lsst.afw.image.ImageF`
1443  The input data structure obtained from Butler.
1444  camera : `lsst.afw.cameraGeom.camera`
1445  The camera associated with the image. Used to find the appropriate
1446  detector.
1447  detectorNum : `int`
1448  The detector this exposure should match.
1449 
1450  Returns
1451  -------
1452  inputExp : `lsst.afw.image.Exposure`
1453  The re-constructed exposure, with appropriate detector parameters.
1454 
1455  Raises
1456  ------
1457  TypeError
1458  Raised if the input data cannot be used to construct an exposure.
1459  """
1460  if isinstance(inputExp, afwImage.DecoratedImageU):
1461  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1462  elif isinstance(inputExp, afwImage.ImageF):
1463  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1464  elif isinstance(inputExp, afwImage.MaskedImageF):
1465  inputExp = afwImage.makeExposure(inputExp)
1466  elif isinstance(inputExp, afwImage.Exposure):
1467  pass
1468  elif inputExp is None:
1469  # Assume this will be caught by the setup if it is a problem.
1470  return inputExp
1471  else:
1472  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1473 
1474  if inputExp.getDetector() is None:
1475  inputExp.setDetector(camera[detectorNum])
1476 
1477  return inputExp
1478 
1479  def convertIntToFloat(self, exposure):
1480  """Convert exposure image from uint16 to float.
1481 
1482  If the exposure does not need to be converted, the input is
1483  immediately returned. For exposures that are converted to use
1484  floating point pixels, the variance is set to unity and the
1485  mask to zero.
1486 
1487  Parameters
1488  ----------
1489  exposure : `lsst.afw.image.Exposure`
1490  The raw exposure to be converted.
1491 
1492  Returns
1493  -------
1494  newexposure : `lsst.afw.image.Exposure`
1495  The input ``exposure``, converted to floating point pixels.
1496 
1497  Raises
1498  ------
1499  RuntimeError
1500  Raised if the exposure type cannot be converted to float.
1501 
1502  """
1503  if isinstance(exposure, afwImage.ExposureF):
1504  # Nothing to be done
1505  return exposure
1506  if not hasattr(exposure, "convertF"):
1507  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1508 
1509  newexposure = exposure.convertF()
1510  newexposure.variance[:] = 1
1511  newexposure.mask[:] = 0x0
1512 
1513  return newexposure
1514 
1515  def maskAmplifier(self, ccdExposure, amp, defects):
1516  """Identify bad amplifiers, saturated and suspect pixels.
1517 
1518  Parameters
1519  ----------
1520  ccdExposure : `lsst.afw.image.Exposure`
1521  Input exposure to be masked.
1522  amp : `lsst.afw.table.AmpInfoCatalog`
1523  Catalog of parameters defining the amplifier on this
1524  exposure to mask.
1525  defects : `lsst.meas.algorithms.Defects`
1526  List of defects. Used to determine if the entire
1527  amplifier is bad.
1528 
1529  Returns
1530  -------
1531  badAmp : `Bool`
1532  If this is true, the entire amplifier area is covered by
1533  defects and unusable.
1534 
1535  """
1536  maskedImage = ccdExposure.getMaskedImage()
1537 
1538  badAmp = False
1539 
1540  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1541  # comparison with current defects definition.
1542  if defects is not None:
1543  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1544 
1545  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1546  # association with pixels in current ccdExposure).
1547  if badAmp:
1548  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1549  afwImage.PARENT)
1550  maskView = dataView.getMask()
1551  maskView |= maskView.getPlaneBitMask("BAD")
1552  del maskView
1553  return badAmp
1554 
1555  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1556  # Saturation and suspect pixels can be masked now, though.
1557  limits = dict()
1558  if self.config.doSaturation and not badAmp:
1559  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1560  if self.config.doSuspect and not badAmp:
1561  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1562  if math.isfinite(self.config.saturation):
1563  limits.update({self.config.saturatedMaskName: self.config.saturation})
1564 
1565  for maskName, maskThreshold in limits.items():
1566  if not math.isnan(maskThreshold):
1567  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1568  isrFunctions.makeThresholdMask(
1569  maskedImage=dataView,
1570  threshold=maskThreshold,
1571  growFootprints=0,
1572  maskName=maskName
1573  )
1574 
1575  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1576  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1577  afwImage.PARENT)
1578  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1579  self.config.suspectMaskName])
1580  if numpy.all(maskView.getArray() & maskVal > 0):
1581  badAmp = True
1582 
1583  return badAmp
1584 
1585  def overscanCorrection(self, ccdExposure, amp):
1586  """Apply overscan correction in place.
1587 
1588  This method does initial pixel rejection of the overscan
1589  region. The overscan can also be optionally segmented to
1590  allow for discontinuous overscan responses to be fit
1591  separately. The actual overscan subtraction is performed by
1592  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1593  which is called here after the amplifier is preprocessed.
1594 
1595  Parameters
1596  ----------
1597  ccdExposure : `lsst.afw.image.Exposure`
1598  Exposure to have overscan correction performed.
1599  amp : `lsst.afw.table.AmpInfoCatalog`
1600  The amplifier to consider while correcting the overscan.
1601 
1602  Returns
1603  -------
1604  overscanResults : `lsst.pipe.base.Struct`
1605  Result struct with components:
1606  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1607  Value or fit subtracted from the amplifier image data.
1608  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1609  Value or fit subtracted from the overscan image data.
1610  - ``overscanImage`` : `lsst.afw.image.Image`
1611  Image of the overscan region with the overscan
1612  correction applied. This quantity is used to estimate
1613  the amplifier read noise empirically.
1614 
1615  Raises
1616  ------
1617  RuntimeError
1618  Raised if the ``amp`` does not contain raw pixel information.
1619 
1620  See Also
1621  --------
1622  lsst.ip.isr.isrFunctions.overscanCorrection
1623  """
1624  if not amp.getHasRawInfo():
1625  raise RuntimeError("This method must be executed on an amp with raw information.")
1626 
1627  if amp.getRawHorizontalOverscanBBox().isEmpty():
1628  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1629  return None
1630 
1631  statControl = afwMath.StatisticsControl()
1632  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1633 
1634  # Determine the bounding boxes
1635  dataBBox = amp.getRawDataBBox()
1636  oscanBBox = amp.getRawHorizontalOverscanBBox()
1637  dx0 = 0
1638  dx1 = 0
1639 
1640  prescanBBox = amp.getRawPrescanBBox()
1641  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1642  dx0 += self.config.overscanNumLeadingColumnsToSkip
1643  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1644  else:
1645  dx0 += self.config.overscanNumTrailingColumnsToSkip
1646  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1647 
1648  # Determine if we need to work on subregions of the amplifier and overscan.
1649  imageBBoxes = []
1650  overscanBBoxes = []
1651 
1652  if ((self.config.overscanBiasJump and
1653  self.config.overscanBiasJumpLocation) and
1654  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1655  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1656  self.config.overscanBiasJumpDevices)):
1657  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1658  yLower = self.config.overscanBiasJumpLocation
1659  yUpper = dataBBox.getHeight() - yLower
1660  else:
1661  yUpper = self.config.overscanBiasJumpLocation
1662  yLower = dataBBox.getHeight() - yUpper
1663 
1664  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1665  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1666  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1667  lsst.geom.Extent2I(dx0, 0),
1668  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1669  yLower)))
1670 
1671  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1672  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1673  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1674  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1675  yUpper)))
1676  else:
1677  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1678  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1679  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1680  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1681  oscanBBox.getHeight())))
1682 
1683  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1684  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1685  ampImage = ccdExposure.maskedImage[imageBBox]
1686  overscanImage = ccdExposure.maskedImage[overscanBBox]
1687 
1688  overscanArray = overscanImage.image.array
1689  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1690  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1691  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1692 
1693  statControl = afwMath.StatisticsControl()
1694  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1695 
1696  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1697  overscanImage=overscanImage,
1698  fitType=self.config.overscanFitType,
1699  order=self.config.overscanOrder,
1700  collapseRej=self.config.overscanNumSigmaClip,
1701  statControl=statControl,
1702  overscanIsInt=self.config.overscanIsInt
1703  )
1704 
1705  # Measure average overscan levels and record them in the metadata
1706  levelStat = afwMath.MEDIAN
1707  sigmaStat = afwMath.STDEVCLIP
1708 
1709  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1710  self.config.qa.flatness.nIter)
1711  metadata = ccdExposure.getMetadata()
1712  ampNum = amp.getName()
1713  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1714  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1715  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1716  else:
1717  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1718  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1719  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1720 
1721  return overscanResults
1722 
1723  def updateVariance(self, ampExposure, amp, overscanImage=None):
1724  """Set the variance plane using the amplifier gain and read noise
1725 
1726  The read noise is calculated from the ``overscanImage`` if the
1727  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1728  the value from the amplifier data is used.
1729 
1730  Parameters
1731  ----------
1732  ampExposure : `lsst.afw.image.Exposure`
1733  Exposure to process.
1734  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1735  Amplifier detector data.
1736  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1737  Image of overscan, required only for empirical read noise.
1738 
1739  See also
1740  --------
1741  lsst.ip.isr.isrFunctions.updateVariance
1742  """
1743  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1744  gain = amp.getGain()
1745 
1746  if math.isnan(gain):
1747  gain = 1.0
1748  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1749  elif gain <= 0:
1750  patchedGain = 1.0
1751  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1752  (amp.getName(), gain, patchedGain))
1753  gain = patchedGain
1754 
1755  if self.config.doEmpiricalReadNoise and overscanImage is None:
1756  self.log.info("Overscan is none for EmpiricalReadNoise")
1757 
1758  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1759  stats = afwMath.StatisticsControl()
1760  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1761  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1762  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1763  else:
1764  readNoise = amp.getReadNoise()
1765 
1766  isrFunctions.updateVariance(
1767  maskedImage=ampExposure.getMaskedImage(),
1768  gain=gain,
1769  readNoise=readNoise,
1770  )
1771 
1772  def darkCorrection(self, exposure, darkExposure, invert=False):
1773  """!Apply dark correction in place.
1774 
1775  Parameters
1776  ----------
1777  exposure : `lsst.afw.image.Exposure`
1778  Exposure to process.
1779  darkExposure : `lsst.afw.image.Exposure`
1780  Dark exposure of the same size as ``exposure``.
1781  invert : `Bool`, optional
1782  If True, re-add the dark to an already corrected image.
1783 
1784  Raises
1785  ------
1786  RuntimeError
1787  Raised if either ``exposure`` or ``darkExposure`` do not
1788  have their dark time defined.
1789 
1790  See Also
1791  --------
1792  lsst.ip.isr.isrFunctions.darkCorrection
1793  """
1794  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1795  if math.isnan(expScale):
1796  raise RuntimeError("Exposure darktime is NAN")
1797  if darkExposure.getInfo().getVisitInfo() is not None:
1798  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1799  else:
1800  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1801  # so getDarkTime() does not exist.
1802  darkScale = 1.0
1803 
1804  if math.isnan(darkScale):
1805  raise RuntimeError("Dark calib darktime is NAN")
1806  isrFunctions.darkCorrection(
1807  maskedImage=exposure.getMaskedImage(),
1808  darkMaskedImage=darkExposure.getMaskedImage(),
1809  expScale=expScale,
1810  darkScale=darkScale,
1811  invert=invert,
1812  trimToFit=self.config.doTrimToMatchCalib
1813  )
1814 
1815  def doLinearize(self, detector):
1816  """!Check if linearization is needed for the detector cameraGeom.
1817 
1818  Checks config.doLinearize and the linearity type of the first
1819  amplifier.
1820 
1821  Parameters
1822  ----------
1823  detector : `lsst.afw.cameraGeom.Detector`
1824  Detector to get linearity type from.
1825 
1826  Returns
1827  -------
1828  doLinearize : `Bool`
1829  If True, linearization should be performed.
1830  """
1831  return self.config.doLinearize and \
1832  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1833 
1834  def flatCorrection(self, exposure, flatExposure, invert=False):
1835  """!Apply flat correction in place.
1836 
1837  Parameters
1838  ----------
1839  exposure : `lsst.afw.image.Exposure`
1840  Exposure to process.
1841  flatExposure : `lsst.afw.image.Exposure`
1842  Flat exposure of the same size as ``exposure``.
1843  invert : `Bool`, optional
1844  If True, unflatten an already flattened image.
1845 
1846  See Also
1847  --------
1848  lsst.ip.isr.isrFunctions.flatCorrection
1849  """
1850  isrFunctions.flatCorrection(
1851  maskedImage=exposure.getMaskedImage(),
1852  flatMaskedImage=flatExposure.getMaskedImage(),
1853  scalingType=self.config.flatScalingType,
1854  userScale=self.config.flatUserScale,
1855  invert=invert,
1856  trimToFit=self.config.doTrimToMatchCalib
1857  )
1858 
1859  def saturationDetection(self, exposure, amp):
1860  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1861 
1862  Parameters
1863  ----------
1864  exposure : `lsst.afw.image.Exposure`
1865  Exposure to process. Only the amplifier DataSec is processed.
1866  amp : `lsst.afw.table.AmpInfoCatalog`
1867  Amplifier detector data.
1868 
1869  See Also
1870  --------
1871  lsst.ip.isr.isrFunctions.makeThresholdMask
1872  """
1873  if not math.isnan(amp.getSaturation()):
1874  maskedImage = exposure.getMaskedImage()
1875  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1876  isrFunctions.makeThresholdMask(
1877  maskedImage=dataView,
1878  threshold=amp.getSaturation(),
1879  growFootprints=0,
1880  maskName=self.config.saturatedMaskName,
1881  )
1882 
1883  def saturationInterpolation(self, ccdExposure):
1884  """!Interpolate over saturated pixels, in place.
1885 
1886  This method should be called after `saturationDetection`, to
1887  ensure that the saturated pixels have been identified in the
1888  SAT mask. It should also be called after `assembleCcd`, since
1889  saturated regions may cross amplifier boundaries.
1890 
1891  Parameters
1892  ----------
1893  exposure : `lsst.afw.image.Exposure`
1894  Exposure to process.
1895 
1896  See Also
1897  --------
1898  lsst.ip.isr.isrTask.saturationDetection
1899  lsst.ip.isr.isrFunctions.interpolateFromMask
1900  """
1901  isrFunctions.interpolateFromMask(
1902  maskedImage=ccdExposure.getMaskedImage(),
1903  fwhm=self.config.fwhm,
1904  growFootprints=self.config.growSaturationFootprintSize,
1905  maskName=self.config.saturatedMaskName,
1906  )
1907 
1908  def suspectDetection(self, exposure, amp):
1909  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1910 
1911  Parameters
1912  ----------
1913  exposure : `lsst.afw.image.Exposure`
1914  Exposure to process. Only the amplifier DataSec is processed.
1915  amp : `lsst.afw.table.AmpInfoCatalog`
1916  Amplifier detector data.
1917 
1918  See Also
1919  --------
1920  lsst.ip.isr.isrFunctions.makeThresholdMask
1921 
1922  Notes
1923  -----
1924  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1925  This is intended to indicate pixels that may be affected by unknown systematics;
1926  for example if non-linearity corrections above a certain level are unstable
1927  then that would be a useful value for suspectLevel. A value of `nan` indicates
1928  that no such level exists and no pixels are to be masked as suspicious.
1929  """
1930  suspectLevel = amp.getSuspectLevel()
1931  if math.isnan(suspectLevel):
1932  return
1933 
1934  maskedImage = exposure.getMaskedImage()
1935  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1936  isrFunctions.makeThresholdMask(
1937  maskedImage=dataView,
1938  threshold=suspectLevel,
1939  growFootprints=0,
1940  maskName=self.config.suspectMaskName,
1941  )
1942 
1943  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1944  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1945 
1946  Parameters
1947  ----------
1948  ccdExposure : `lsst.afw.image.Exposure`
1949  Exposure to process.
1950  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
1951  `lsst.afw.image.DefectBase`.
1952  List of defects to mask and interpolate.
1953 
1954  Notes
1955  -----
1956  Call this after CCD assembly, since defects may cross amplifier boundaries.
1957  """
1958  maskedImage = ccdExposure.getMaskedImage()
1959  if not isinstance(defectBaseList, Defects):
1960  # Promotes DefectBase to Defect
1961  defectList = Defects(defectBaseList)
1962  else:
1963  defectList = defectBaseList
1964  defectList.maskPixels(maskedImage, maskName="BAD")
1965  isrFunctions.interpolateDefectList(
1966  maskedImage=maskedImage,
1967  defectList=defectList,
1968  fwhm=self.config.fwhm,
1969  )
1970 
1971  if self.config.numEdgeSuspect > 0:
1972  goodBBox = maskedImage.getBBox()
1973  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1974  goodBBox.grow(-self.config.numEdgeSuspect)
1975  # Mask pixels outside goodBBox as SUSPECT
1976  SourceDetectionTask.setEdgeBits(
1977  maskedImage,
1978  goodBBox,
1979  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1980  )
1981 
1982  def maskAndInterpNan(self, exposure):
1983  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1984 
1985  Parameters
1986  ----------
1987  exposure : `lsst.afw.image.Exposure`
1988  Exposure to process.
1989 
1990  Notes
1991  -----
1992  We mask and interpolate over all NaNs, including those
1993  that are masked with other bits (because those may or may
1994  not be interpolated over later, and we want to remove all
1995  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1996  is used to preserve the historical name.
1997  """
1998  maskedImage = exposure.getMaskedImage()
1999 
2000  # Find and mask NaNs
2001  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2002  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2003  numNans = maskNans(maskedImage, maskVal)
2004  self.metadata.set("NUMNANS", numNans)
2005 
2006  # Interpolate over these previously-unmasked NaNs
2007  if numNans > 0:
2008  self.log.warn("There were %i unmasked NaNs", numNans)
2009  nanDefectList = Defects.fromMask(
2010  maskedImage=maskedImage,
2011  maskName='UNMASKEDNAN',
2012  )
2013  isrFunctions.interpolateDefectList(
2014  maskedImage=exposure.getMaskedImage(),
2015  defectList=nanDefectList,
2016  fwhm=self.config.fwhm,
2017  )
2018 
2019  def measureBackground(self, exposure, IsrQaConfig=None):
2020  """Measure the image background in subgrids, for quality control purposes.
2021 
2022  Parameters
2023  ----------
2024  exposure : `lsst.afw.image.Exposure`
2025  Exposure to process.
2026  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2027  Configuration object containing parameters on which background
2028  statistics and subgrids to use.
2029  """
2030  if IsrQaConfig is not None:
2031  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2032  IsrQaConfig.flatness.nIter)
2033  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2034  statsControl.setAndMask(maskVal)
2035  maskedImage = exposure.getMaskedImage()
2036  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2037  skyLevel = stats.getValue(afwMath.MEDIAN)
2038  skySigma = stats.getValue(afwMath.STDEVCLIP)
2039  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
2040  metadata = exposure.getMetadata()
2041  metadata.set('SKYLEVEL', skyLevel)
2042  metadata.set('SKYSIGMA', skySigma)
2043 
2044  # calcluating flatlevel over the subgrids
2045  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2046  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2047  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2048  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2049  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2050  skyLevels = numpy.zeros((nX, nY))
2051 
2052  for j in range(nY):
2053  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2054  for i in range(nX):
2055  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2056 
2057  xLLC = xc - meshXHalf
2058  yLLC = yc - meshYHalf
2059  xURC = xc + meshXHalf - 1
2060  yURC = yc + meshYHalf - 1
2061 
2062  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2063  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2064 
2065  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2066 
2067  good = numpy.where(numpy.isfinite(skyLevels))
2068  skyMedian = numpy.median(skyLevels[good])
2069  flatness = (skyLevels[good] - skyMedian) / skyMedian
2070  flatness_rms = numpy.std(flatness)
2071  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2072 
2073  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2074  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2075  (nX, nY, flatness_pp, flatness_rms))
2076 
2077  metadata.set('FLATNESS_PP', float(flatness_pp))
2078  metadata.set('FLATNESS_RMS', float(flatness_rms))
2079  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2080  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2081  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2082 
2083  def roughZeroPoint(self, exposure):
2084  """Set an approximate magnitude zero point for the exposure.
2085 
2086  Parameters
2087  ----------
2088  exposure : `lsst.afw.image.Exposure`
2089  Exposure to process.
2090  """
2091  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2092  if filterName in self.config.fluxMag0T1:
2093  fluxMag0 = self.config.fluxMag0T1[filterName]
2094  else:
2095  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2096  fluxMag0 = self.config.defaultFluxMag0T1
2097 
2098  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2099  if not expTime > 0: # handle NaN as well as <= 0
2100  self.log.warn("Non-positive exposure time; skipping rough zero point")
2101  return
2102 
2103  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2104  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2105 
2106  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2107  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2108 
2109  Parameters
2110  ----------
2111  ccdExposure : `lsst.afw.image.Exposure`
2112  Exposure to process.
2113  fpPolygon : `lsst.afw.geom.Polygon`
2114  Polygon in focal plane coordinates.
2115  """
2116  # Get ccd corners in focal plane coordinates
2117  ccd = ccdExposure.getDetector()
2118  fpCorners = ccd.getCorners(FOCAL_PLANE)
2119  ccdPolygon = Polygon(fpCorners)
2120 
2121  # Get intersection of ccd corners with fpPolygon
2122  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2123 
2124  # Transform back to pixel positions and build new polygon
2125  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2126  validPolygon = Polygon(ccdPoints)
2127  ccdExposure.getInfo().setValidPolygon(validPolygon)
2128 
2129  @contextmanager
2130  def flatContext(self, exp, flat, dark=None):
2131  """Context manager that applies and removes flats and darks,
2132  if the task is configured to apply them.
2133 
2134  Parameters
2135  ----------
2136  exp : `lsst.afw.image.Exposure`
2137  Exposure to process.
2138  flat : `lsst.afw.image.Exposure`
2139  Flat exposure the same size as ``exp``.
2140  dark : `lsst.afw.image.Exposure`, optional
2141  Dark exposure the same size as ``exp``.
2142 
2143  Yields
2144  ------
2145  exp : `lsst.afw.image.Exposure`
2146  The flat and dark corrected exposure.
2147  """
2148  if self.config.doDark and dark is not None:
2149  self.darkCorrection(exp, dark)
2150  if self.config.doFlat:
2151  self.flatCorrection(exp, flat)
2152  try:
2153  yield exp
2154  finally:
2155  if self.config.doFlat:
2156  self.flatCorrection(exp, flat, invert=True)
2157  if self.config.doDark and dark is not None:
2158  self.darkCorrection(exp, dark, invert=True)
2159 
2160  def debugView(self, exposure, stepname):
2161  """Utility function to examine ISR exposure at different stages.
2162 
2163  Parameters
2164  ----------
2165  exposure : `lsst.afw.image.Exposure`
2166  Exposure to view.
2167  stepname : `str`
2168  State of processing to view.
2169  """
2170  frame = getDebugFrame(self._display, stepname)
2171  if frame:
2172  display = getDisplay(frame)
2173  display.scale('asinh', 'zscale')
2174  display.mtv(exposure)
2175 
2176 
2177 class FakeAmp(object):
2178  """A Detector-like object that supports returning gain and saturation level
2179 
2180  This is used when the input exposure does not have a detector.
2181 
2182  Parameters
2183  ----------
2184  exposure : `lsst.afw.image.Exposure`
2185  Exposure to generate a fake amplifier for.
2186  config : `lsst.ip.isr.isrTaskConfig`
2187  Configuration to apply to the fake amplifier.
2188  """
2189 
2190  def __init__(self, exposure, config):
2191  self._bbox = exposure.getBBox(afwImage.LOCAL)
2193  self._gain = config.gain
2194  self._readNoise = config.readNoise
2195  self._saturation = config.saturation
2196 
2197  def getBBox(self):
2198  return self._bbox
2199 
2200  def getRawBBox(self):
2201  return self._bbox
2202 
2203  def getHasRawInfo(self):
2204  return True # but see getRawHorizontalOverscanBBox()
2205 
2207  return self._RawHorizontalOverscanBBox
2208 
2209  def getGain(self):
2210  return self._gain
2211 
2212  def getReadNoise(self):
2213  return self._readNoise
2214 
2215  def getSaturation(self):
2216  return self._saturation
2217 
2218  def getSuspectLevel(self):
2219  return float("NaN")
2220 
2221 
2222 class RunIsrConfig(pexConfig.Config):
2223  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2224 
2225 
2226 class RunIsrTask(pipeBase.CmdLineTask):
2227  """Task to wrap the default IsrTask to allow it to be retargeted.
2228 
2229  The standard IsrTask can be called directly from a command line
2230  program, but doing so removes the ability of the task to be
2231  retargeted. As most cameras override some set of the IsrTask
2232  methods, this would remove those data-specific methods in the
2233  output post-ISR images. This wrapping class fixes the issue,
2234  allowing identical post-ISR images to be generated by both the
2235  processCcd and isrTask code.
2236  """
2237  ConfigClass = RunIsrConfig
2238  _DefaultName = "runIsr"
2239 
2240  def __init__(self, *args, **kwargs):
2241  super().__init__(*args, **kwargs)
2242  self.makeSubtask("isr")
2243 
2244  def runDataRef(self, dataRef):
2245  """
2246  Parameters
2247  ----------
2248  dataRef : `lsst.daf.persistence.ButlerDataRef`
2249  data reference of the detector data to be processed
2250 
2251  Returns
2252  -------
2253  result : `pipeBase.Struct`
2254  Result struct with component:
2255 
2256  - exposure : `lsst.afw.image.Exposure`
2257  Post-ISR processed exposure.
2258  """
2259  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:710
def runDataRef(self, sensorRef)
Definition: isrTask.py:1339
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2019
def debugView(self, exposure, stepname)
Definition: isrTask.py:2160
def __init__(self, kwargs)
Definition: isrTask.py:700
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1432
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:817
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:777
def runDataRef(self, dataRef)
Definition: isrTask.py:2244
def __init__(self, args, kwargs)
Definition: isrTask.py:2240
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1982
def getPrerequisiteDatasetTypes(cls, config)
Definition: isrTask.py:760
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1883
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2083
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2206
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:747
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1585
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1479
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1834
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:814
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1391
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1772
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1815
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2106
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1515
def getPerDatasetTypeDimensions(cls, config)
Definition: isrTask.py:770
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:955
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2130
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1723
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1908
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1943
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1859
def __init__(self, exposure, config)
Definition: isrTask.py:2190