lsst.ip.isr  16.0-19-gade7dc4
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.afw.table as afwTable
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 
33 from contextlib import contextmanager
34 from lsstDebug import getDebugFrame
35 
36 from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, NullLinearityType
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.meas.algorithms.detection import SourceDetectionTask
41 from lsst.meas.algorithms import Defect
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 
47 from .assembleCcdTask import AssembleCcdTask
48 from .crosstalk import CrosstalkTask
49 from .fringe import FringeTask
50 from .isr import maskNans
51 from .masking import MaskingTask
52 from .straylight import StrayLightTask
53 from .vignette import VignetteTask
54 
55 __all__ = ["IsrTask", "RunIsrTask"]
56 
57 
58 class IsrTaskConfig(pexConfig.Config):
59  """Configuration parameters for IsrTask.
60 
61  Items are grouped in the order in which they are executed by the task.
62  """
63  # General ISR configuration
64 
65  # gen3 options
66  isrName = pexConfig.Field(
67  dtype=str,
68  doc="Name of ISR",
69  default="ISR",
70  )
71 
72  # input datasets
73  ccdExposure = pipeBase.InputDatasetField(
74  doc="Input exposure to process",
75  name="raw",
76  scalar=True,
77  storageClass="ExposureU",
78  dimensions=["Instrument", "Exposure", "Detector"],
79  )
80  camera = pipeBase.InputDatasetField(
81  doc="Input camera to construct complete exposures.",
82  name="camera",
83  scalar=True,
84  storageClass="TablePersistableCamera",
85  dimensions=["Instrument", "ExposureRange"],
86  )
87  bias = pipeBase.InputDatasetField(
88  doc="Input bias calibration.",
89  name="bias",
90  scalar=True,
91  storageClass="ImageF",
92  dimensions=["Instrument", "ExposureRange", "Detector"],
93  )
94  dark = pipeBase.InputDatasetField(
95  doc="Input dark calibration.",
96  name="dark",
97  scalar=True,
98  storageClass="ImageF",
99  dimensions=["Instrument", "ExposureRange", "Detector"],
100  )
101  flat = pipeBase.InputDatasetField(
102  doc="Input flat calibration.",
103  name="flat",
104  scalar=True,
105  storageClass="MaskedImageF",
106  dimensions=["Instrument", "PhysicalFilter", "ExposureRange", "Detector"],
107  )
108  bfKernel = pipeBase.InputDatasetField(
109  doc="Input brighter-fatter kernel.",
110  name="bfKernel",
111  scalar=True,
112  storageClass="NumpyArray",
113  dimensions=["Instrument", "ExposureRange"],
114  )
115  defects = pipeBase.InputDatasetField(
116  doc="Input defect tables.",
117  name="defects",
118  scalar=True,
119  storageClass="Catalog",
120  dimensions=["Instrument", "ExposureRange", "Detector"],
121  )
122  opticsTransmission = pipeBase.InputDatasetField(
123  doc="Transmission curve due to the optics.",
124  name="transmission_optics",
125  scalar=True,
126  storageClass="TablePersistableTransmissionCurve",
127  dimensions=["Instrument", "ExposureRange"],
128  )
129  filterTransmission = pipeBase.InputDatasetField(
130  doc="Transmission curve due to the filter.",
131  name="transmission_filter",
132  scalar=True,
133  storageClass="TablePersistableTransmissionCurve",
134  dimensions=["Instrument", "PhysicalFilter", "ExposureRange"],
135  )
136  sensorTransmission = pipeBase.InputDatasetField(
137  doc="Transmission curve due to the sensor.",
138  name="transmission_sensor",
139  scalar=True,
140  storageClass="TablePersistableTransmissionCurve",
141  dimensions=["Instrument", "ExposureRange", "Detector"],
142  )
143  atmosphereTransmission = pipeBase.InputDatasetField(
144  doc="Transmission curve due to the atmosphere.",
145  name="transmission_atmosphere",
146  scalar=True,
147  storageClass="TablePersistableTransmissionCurve",
148  dimensions=["Instrument"],
149  )
150 
151  # output datasets
152  outputExposure = pipeBase.OutputDatasetField(
153  doc="Output ISR processed exposure.",
154  name="postISRCCD",
155  scalar=True,
156  storageClass="ExposureF",
157  dimensions=["Instrument", "Visit", "Detector"],
158  )
159  outputOssThumbnail = pipeBase.OutputDatasetField(
160  doc="Output Overscan-subtracted thumbnail image.",
161  name="OssThumb",
162  scalar=True,
163  storageClass="Thumbnail",
164  dimensions=["Instrument", "Visit", "Detector"],
165  )
166  outputFlattenedThumbnail = pipeBase.OutputDatasetField(
167  doc="Output flat-corrected thumbnail image.",
168  name="FlattenedThumb",
169  scalar=True,
170  storageClass="TextStorage",
171  dimensions=["Instrument", "Visit", "Detector"],
172  )
173 
174  quantum = pipeBase.QuantumConfig(
175  dimensions=["Visit", "Detector", "Instrument"],
176  )
177 
178 
179  datasetType = pexConfig.Field(
180  dtype=str,
181  doc="Dataset type for input data; users will typically leave this alone, "
182  "but camera-specific ISR tasks will override it",
183  default="raw",
184  )
185 
186  fallbackFilterName = pexConfig.Field(
187  dtype=str,
188  doc="Fallback default filter name for calibrations.",
189  optional=True
190  )
191  expectWcs = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
195  )
196  fwhm = pexConfig.Field(
197  dtype=float,
198  doc="FWHM of PSF in arcseconds.",
199  default=1.0,
200  )
201  qa = pexConfig.ConfigField(
202  dtype=isrQa.IsrQaConfig,
203  doc="QA related configuration options.",
204  )
205 
206  # Image conversion configuration
207  doConvertIntToFloat = pexConfig.Field(
208  dtype=bool,
209  doc="Convert integer raw images to floating point values?",
210  default=True,
211  )
212 
213  # Saturated pixel handling.
214  doSaturation = pexConfig.Field(
215  dtype=bool,
216  doc="Mask saturated pixels?",
217  default=True,
218  )
219  saturatedMaskName = pexConfig.Field(
220  dtype=str,
221  doc="Name of mask plane to use in saturation detection and interpolation",
222  default="SAT",
223  )
224  saturation = pexConfig.Field(
225  dtype=float,
226  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
227  default=float("NaN"),
228  )
229  growSaturationFootprintSize = pexConfig.Field(
230  dtype=int,
231  doc="Number of pixels by which to grow the saturation footprints",
232  default=1,
233  )
234 
235  # Suspect pixel handling.
236  doSuspect = pexConfig.Field(
237  dtype=bool,
238  doc="Mask suspect pixels?",
239  default=True,
240  )
241  suspectMaskName = pexConfig.Field(
242  dtype=str,
243  doc="Name of mask plane to use for suspect pixels",
244  default="SUSPECT",
245  )
246  numEdgeSuspect = pexConfig.Field(
247  dtype=int,
248  doc="Number of edge pixels to be flagged as untrustworthy.",
249  default=0,
250  )
251 
252  # Initial masking options.
253  doSetBadRegions = pexConfig.Field(
254  dtype=bool,
255  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
256  default=True,
257  )
258  badStatistic = pexConfig.ChoiceField(
259  dtype=str,
260  doc="How to estimate the average value for BAD regions.",
261  default='MEANCLIP',
262  allowed={
263  "MEANCLIP": "Correct using the (clipped) mean of good data",
264  "MEDIAN": "Correct using the median of the good data",
265  },
266  )
267 
268  # Overscan subtraction configuration.
269  doOverscan = pexConfig.Field(
270  dtype=bool,
271  doc="Do overscan subtraction?",
272  default=True,
273  )
274  overscanFitType = pexConfig.ChoiceField(
275  dtype=str,
276  doc="The method for fitting the overscan bias level.",
277  default='MEDIAN',
278  allowed={
279  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
280  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
281  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
282  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
283  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
284  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
285  "MEAN": "Correct using the mean of the overscan region",
286  "MEANCLIP": "Correct using a clipped mean of the overscan region",
287  "MEDIAN": "Correct using the median of the overscan region",
288  },
289  )
290  overscanOrder = pexConfig.Field(
291  dtype=int,
292  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
293  "or number of spline knots if overscan fit type is a spline."),
294  default=1,
295  )
296  overscanNumSigmaClip = pexConfig.Field(
297  dtype=float,
298  doc="Rejection threshold (sigma) for collapsing overscan before fit",
299  default=3.0,
300  )
301  overscanIsInt = pexConfig.Field(
302  dtype=bool,
303  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
304  default=True,
305  )
306  overscanNumLeadingColumnsToSkip = pexConfig.Field(
307  dtype=int,
308  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
309  default=0,
310  )
311  overscanNumTrailingColumnsToSkip = pexConfig.Field(
312  dtype=int,
313  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
314  default=0,
315  )
316  overscanMaxDev = pexConfig.Field(
317  dtype=float,
318  doc="Maximum deviation from the median for overscan",
319  default=1000.0, check=lambda x: x > 0
320  )
321  overscanBiasJump = pexConfig.Field(
322  dtype=bool,
323  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
324  default=False,
325  )
326  overscanBiasJumpKeyword = pexConfig.Field(
327  dtype=str,
328  doc="Header keyword containing information about devices.",
329  default="NO_SUCH_KEY",
330  )
331  overscanBiasJumpDevices = pexConfig.ListField(
332  dtype=str,
333  doc="List of devices that need piecewise overscan correction.",
334  default=(),
335  )
336  overscanBiasJumpLocation = pexConfig.Field(
337  dtype=int,
338  doc="Location of bias jump along y-axis.",
339  default=0,
340  )
341 
342  # Amplifier to CCD assembly configuration
343  doAssembleCcd = pexConfig.Field(
344  dtype=bool,
345  default=True,
346  doc="Assemble amp-level exposures into a ccd-level exposure?"
347  )
348  assembleCcd = pexConfig.ConfigurableField(
349  target=AssembleCcdTask,
350  doc="CCD assembly task",
351  )
352 
353  # General calibration configuration.
354  doAssembleIsrExposures = pexConfig.Field(
355  dtype=bool,
356  default=False,
357  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
358  )
359  doTrimToMatchCalib = pexConfig.Field(
360  dtype=bool,
361  default=False,
362  doc="Trim raw data to match calibration bounding boxes?"
363  )
364 
365  # Bias subtraction.
366  doBias = pexConfig.Field(
367  dtype=bool,
368  doc="Apply bias frame correction?",
369  default=True,
370  )
371  biasDataProductName = pexConfig.Field(
372  dtype=str,
373  doc="Name of the bias data product",
374  default="bias",
375  )
376 
377  # Variance construction
378  doVariance = pexConfig.Field(
379  dtype=bool,
380  doc="Calculate variance?",
381  default=True
382  )
383  gain = pexConfig.Field(
384  dtype=float,
385  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
386  default=float("NaN"),
387  )
388  readNoise = pexConfig.Field(
389  dtype=float,
390  doc="The read noise to use if no Detector is present in the Exposure",
391  default=0.0,
392  )
393  doEmpiricalReadNoise = pexConfig.Field(
394  dtype=bool,
395  default=False,
396  doc="Calculate empirical read noise instead of value from AmpInfo data?"
397  )
398 
399  # Linearization.
400  doLinearize = pexConfig.Field(
401  dtype=bool,
402  doc="Correct for nonlinearity of the detector's response?",
403  default=True,
404  )
405 
406  # Crosstalk.
407  doCrosstalk = pexConfig.Field(
408  dtype=bool,
409  doc="Apply intra-CCD crosstalk correction?",
410  default=False,
411  )
412  doCrosstalkBeforeAssemble = pexConfig.Field(
413  dtype=bool,
414  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
415  default=True,
416  )
417  crosstalk = pexConfig.ConfigurableField(
418  target=CrosstalkTask,
419  doc="Intra-CCD crosstalk correction",
420  )
421 
422  # Masking option prior to brighter-fatter.
423  doWidenSaturationTrails = pexConfig.Field(
424  dtype=bool,
425  doc="Widen bleed trails based on their width?",
426  default=True
427  )
428 
429  # Brighter-Fatter correction.
430  doBrighterFatter = pexConfig.Field(
431  dtype=bool,
432  default=False,
433  doc="Apply the brighter fatter correction"
434  )
435  brighterFatterLevel = pexConfig.ChoiceField(
436  dtype=str,
437  default="DETECTOR",
438  doc="The level at which to correct for brighter-fatter.",
439  allowed={
440  "AMP": "Every amplifier treated separately.",
441  "DETECTOR": "One kernel per detector",
442  }
443  )
444  brighterFatterKernelFile = pexConfig.Field(
445  dtype=str,
446  default='',
447  doc="Kernel file used for the brighter fatter correction"
448  )
449  brighterFatterMaxIter = pexConfig.Field(
450  dtype=int,
451  default=10,
452  doc="Maximum number of iterations for the brighter fatter correction"
453  )
454  brighterFatterThreshold = pexConfig.Field(
455  dtype=float,
456  default=1000,
457  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
458  " absolute value of the difference between the current corrected image and the one"
459  " from the previous iteration summed over all the pixels."
460  )
461  brighterFatterApplyGain = pexConfig.Field(
462  dtype=bool,
463  default=True,
464  doc="Should the gain be applied when applying the brighter fatter correction?"
465  )
466 
467  # Defect and bad pixel correction options.
468  doDefect = pexConfig.Field(
469  dtype=bool,
470  doc="Apply correction for CCD defects, e.g. hot pixels?",
471  default=True,
472  )
473  doSaturationInterpolation = pexConfig.Field(
474  dtype=bool,
475  doc="Perform interpolation over pixels masked as saturated?",
476  default=True,
477  )
478  numEdgeSuspect = pexConfig.Field(
479  dtype=int,
480  doc="Number of edge pixels to be flagged as untrustworthy.",
481  default=0,
482  )
483 
484  # Dark subtraction.
485  doDark = pexConfig.Field(
486  dtype=bool,
487  doc="Apply dark frame correction?",
488  default=True,
489  )
490  darkDataProductName = pexConfig.Field(
491  dtype=str,
492  doc="Name of the dark data product",
493  default="dark",
494  )
495 
496  # Camera-specific stray light removal.
497  doStrayLight = pexConfig.Field(
498  dtype=bool,
499  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
500  default=False,
501  )
502  strayLight = pexConfig.ConfigurableField(
503  target=StrayLightTask,
504  doc="y-band stray light correction"
505  )
506 
507  # Flat correction.
508  doFlat = pexConfig.Field(
509  dtype=bool,
510  doc="Apply flat field correction?",
511  default=True,
512  )
513  flatDataProductName = pexConfig.Field(
514  dtype=str,
515  doc="Name of the flat data product",
516  default="flat",
517  )
518  flatScalingType = pexConfig.ChoiceField(
519  dtype=str,
520  doc="The method for scaling the flat on the fly.",
521  default='USER',
522  allowed={
523  "USER": "Scale by flatUserScale",
524  "MEAN": "Scale by the inverse of the mean",
525  "MEDIAN": "Scale by the inverse of the median",
526  },
527  )
528  flatUserScale = pexConfig.Field(
529  dtype=float,
530  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
531  default=1.0,
532  )
533  doTweakFlat = pexConfig.Field(
534  dtype=bool,
535  doc="Tweak flats to match observed amplifier ratios?",
536  default=False
537  )
538 
539  # Amplifier normalization based on gains instead of using flats configuration.
540  doApplyGains = pexConfig.Field(
541  dtype=bool,
542  doc="Correct the amplifiers for their gains instead of applying flat correction",
543  default=False,
544  )
545  normalizeGains = pexConfig.Field(
546  dtype=bool,
547  doc="Normalize all the amplifiers in each CCD to have the same median value.",
548  default=False,
549  )
550 
551  # Fringe correction.
552  doFringe = pexConfig.Field(
553  dtype=bool,
554  doc="Apply fringe correction?",
555  default=True,
556  )
557  fringe = pexConfig.ConfigurableField(
558  target=FringeTask,
559  doc="Fringe subtraction task",
560  )
561  fringeAfterFlat = pexConfig.Field(
562  dtype=bool,
563  doc="Do fringe subtraction after flat-fielding?",
564  default=True,
565  )
566 
567  # NAN pixel interpolation option.
568  doNanInterpAfterFlat = pexConfig.Field(
569  dtype=bool,
570  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
571  "also have to interpolate them before flat-fielding."),
572  default=False,
573  )
574 
575  # Distortion model application.
576  doAddDistortionModel = pexConfig.Field(
577  dtype=bool,
578  doc="Apply a distortion model based on camera geometry to the WCS?",
579  default=True,
580  )
581 
582  # Initial CCD-level background statistics options.
583  doMeasureBackground = pexConfig.Field(
584  dtype=bool,
585  doc="Measure the background level on the reduced image?",
586  default=False,
587  )
588 
589  # Camera-specific masking configuration.
590  doCameraSpecificMasking = pexConfig.Field(
591  dtype=bool,
592  doc="Mask camera-specific bad regions?",
593  default=False,
594  )
595  masking = pexConfig.ConfigurableField(
596  target=MaskingTask,
597  doc="Masking task."
598  )
599 
600  # Default photometric calibration options.
601  fluxMag0T1 = pexConfig.DictField(
602  keytype=str,
603  itemtype=float,
604  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
605  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
606  ))
607  )
608  defaultFluxMag0T1 = pexConfig.Field(
609  dtype=float,
610  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
611  default=pow(10.0, 0.4*28.0)
612  )
613 
614  # Vignette correction configuration.
615  doVignette = pexConfig.Field(
616  dtype=bool,
617  doc="Apply vignetting parameters?",
618  default=False,
619  )
620  vignette = pexConfig.ConfigurableField(
621  target=VignetteTask,
622  doc="Vignetting task.",
623  )
624 
625  # Transmission curve configuration.
626  doAttachTransmissionCurve = pexConfig.Field(
627  dtype=bool,
628  default=False,
629  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
630  )
631  doUseOpticsTransmission = pexConfig.Field(
632  dtype=bool,
633  default=True,
634  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
635  )
636  doUseFilterTransmission = pexConfig.Field(
637  dtype=bool,
638  default=True,
639  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
640  )
641  doUseSensorTransmission = pexConfig.Field(
642  dtype=bool,
643  default=True,
644  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
645  )
646  doUseAtmosphereTransmission = pexConfig.Field(
647  dtype=bool,
648  default=True,
649  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
650  )
651 
652  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
653  doWrite = pexConfig.Field(
654  dtype=bool,
655  doc="Persist postISRCCD?",
656  default=True,
657  )
658 
659  def validate(self):
660  super().validate()
661  if self.doFlat and self.doApplyGains:
662  raise ValueError("You may not specify both doFlat and doApplyGains")
663 
664 
665 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
666  r"""Apply common instrument signature correction algorithms to a raw frame.
667 
668  The process for correcting imaging data is very similar from
669  camera to camera. This task provides a vanilla implementation of
670  doing these corrections, including the ability to turn certain
671  corrections off if they are not needed. The inputs to the primary
672  method, `run()`, are a raw exposure to be corrected and the
673  calibration data products. The raw input is a single chip sized
674  mosaic of all amps including overscans and other non-science
675  pixels. The method `runDataRef()` identifies and defines the
676  calibration data products, and is intended for use by a
677  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
678  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
679  subclassed for different camera, although the most camera specific
680  methods have been split into subtasks that can be redirected
681  appropriately.
682 
683  The __init__ method sets up the subtasks for ISR processing, using
684  the defaults from `lsst.ip.isr`.
685 
686  Parameters
687  ----------
688  args : `list`
689  Positional arguments passed to the Task constructor. None used at this time.
690  kwargs : `dict`, optional
691  Keyword arguments passed on to the Task constructor. None used at this time.
692  """
693  ConfigClass = IsrTaskConfig
694  _DefaultName = "isr"
695 
696  def __init__(self, **kwargs):
697  super().__init__(**kwargs)
698  self.makeSubtask("assembleCcd")
699  self.makeSubtask("crosstalk")
700  self.makeSubtask("strayLight")
701  self.makeSubtask("fringe")
702  self.makeSubtask("masking")
703  self.makeSubtask("vignette")
704 
705  @classmethod
706  def getInputDatasetTypes(cls, config):
707  inputTypeDict = super().getInputDatasetTypes(config)
708 
709  # Delete entries from the dictionary of InputDatasetTypes that we know we don't
710  # need because the configuration tells us we will not be bothering with the
711  # correction that uses that IDT.
712  if config.doBias is not True:
713  del inputTypeDict["bias"]
714  if config.doLinearize is not True:
715  del inputTypeDict["linearizer"]
716  if config.doCrosstalk is not True:
717  del inputTypeDict["crosstalkSources"]
718  if config.doBrighterFatter is not True:
719  del inputTypeDict["bfKernel"]
720  if config.doDefect is not True:
721  del inputTypeDict["defects"]
722  if config.doDark is not True:
723  del inputTypeDict["dark"]
724  if config.doFlat is not True:
725  del inputTypeDict["flat"]
726  if config.doAttachTransmissionCurve is not True:
727  del inputTypeDict["opticsTransmission"]
728  del inputTypeDict["filterTransmission"]
729  del inputTypeDict["sensorTransmission"]
730  del inputTypeDict["atmosphereTransmission"]
731  if config.doUseOpticsTransmission is not True:
732  del inputTypeDict["opticsTransmission"]
733  if config.doUseFilterTransmission is not True:
734  del inputTypeDict["filterTransmission"]
735  if config.doUseSensorTransmission is not True:
736  del inputTypeDict["sensorTransmission"]
737  if config.doUseAtmosphereTransmission is not True:
738  del inputTypeDict["atmosphereTransmission"]
739 
740  return inputTypeDict
741 
742  @classmethod
743  def getOutputDatasetTypes(cls, config):
744  outputTypeDict = super().getOutputDatasetTypes(config)
745 
746  if config.qa.doThumbnailOss is not True:
747  del outputTypeDict["outputOssThumbnail"]
748  if config.qa.doThumbnailFlattened is not True:
749  del outputTypeDict["outputFlattenedThumbnail"]
750  if config.doWrite is not True:
751  del outputTypeDict["outputExposure"]
752 
753  return outputTypeDict
754 
755  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
756  try:
757  inputData['detectorNum'] = int(inputDataIds['ccdExposure']['detector'])
758  except Exception as e:
759  raise ValueError(f"Failure to find valid detectorNum value for Dataset {inputDataIds}: {e}")
760 
761  inputData['isGen3'] = True
762 
763  if self.config.doLinearize is True:
764  if 'linearizer' not in inputData.keys():
765  detector = inputData['camera'][inputData['detectorNum']]
766  linearityName = detector.getAmpInfoCatalog()[0].getLinearityType()
767  inputData['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
768 
769  if inputData['defects'] is not None:
770  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
771  # masking expects a list of defects defined by their bounding box
772  defectList = []
773 
774  for r in inputData['defects']:
775  bbox = afwGeom.BoxI(afwGeom.PointI(r.get("x0"), r.get("y0")),
776  afwGeom.ExtentI(r.get("width"), r.get("height")))
777  defectList.append(Defect(bbox))
778 
779  inputData['defects'] = defectList
780 
781  # Broken: DM-17169
782  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
783  # fixed for non-HSC cameras in the future.
784  # inputData['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputDataIds['ccdExposure'])
785  # if self.config.doCrosstalk else None)
786 
787  # Broken: DM-17152
788  # Fringes are not tested to be handled correctly by Gen3 butler.
789  # inputData['fringes'] = (self.fringe.readFringes(inputDataIds['ccdExposure'],
790  # assembler=self.assembleCcd
791  # if self.config.doAssembleIsrExposures else None)
792  # if self.config.doFringe and
793  # self.fringe.checkFilter(inputData['ccdExposure'])
794  # else pipeBase.Struct(fringes=None))
795 
796  return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
797 
798  def makeDatasetType(self, dsConfig):
799  return super().makeDatasetType(dsConfig)
800 
801  def readIsrData(self, dataRef, rawExposure):
802  """!Retrieve necessary frames for instrument signature removal.
803 
804  Pre-fetching all required ISR data products limits the IO
805  required by the ISR. Any conflict between the calibration data
806  available and that needed for ISR is also detected prior to
807  doing processing, allowing it to fail quickly.
808 
809  Parameters
810  ----------
811  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
812  Butler reference of the detector data to be processed
813  rawExposure : `afw.image.Exposure`
814  The raw exposure that will later be corrected with the
815  retrieved calibration data; should not be modified in this
816  method.
817 
818  Returns
819  -------
820  result : `lsst.pipe.base.Struct`
821  Result struct with components (which may be `None`):
822  - ``bias``: bias calibration frame (`afw.image.Exposure`)
823  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
824  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
825  - ``dark``: dark calibration frame (`afw.image.Exposure`)
826  - ``flat``: flat calibration frame (`afw.image.Exposure`)
827  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
828  - ``defects``: list of defects (`list`)
829  - ``fringes``: `lsst.pipe.base.Struct` with components:
830  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
831  - ``seed``: random seed derived from the ccdExposureId for random
832  number generator (`uint32`)
833  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
834  A ``TransmissionCurve`` that represents the throughput of the optics,
835  to be evaluated in focal-plane coordinates.
836  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
837  A ``TransmissionCurve`` that represents the throughput of the filter
838  itself, to be evaluated in focal-plane coordinates.
839  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
840  A ``TransmissionCurve`` that represents the throughput of the sensor
841  itself, to be evaluated in post-assembly trimmed detector coordinates.
842  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
843  A ``TransmissionCurve`` that represents the throughput of the
844  atmosphere, assumed to be spatially constant.
845 
846  """
847  ccd = rawExposure.getDetector()
848  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
849  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
850  if self.config.doBias else None)
851  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
852  linearizer = (dataRef.get("linearizer", immediate=True)
853  if self.doLinearize(ccd) else None)
854  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
855  if self.config.doCrosstalk else None)
856  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
857  if self.config.doDark else None)
858  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName)
859  if self.config.doFlat else None)
860  brighterFatterKernel = (dataRef.get("bfKernel")
861  if self.config.doBrighterFatter else None)
862  defectList = (dataRef.get("defects")
863  if self.config.doDefect else None)
864  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
865  if self.config.doAssembleIsrExposures else None)
866  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
867  else pipeBase.Struct(fringes=None))
868 
869  if self.config.doAttachTransmissionCurve:
870  opticsTransmission = (dataRef.get("transmission_optics")
871  if self.config.doUseOpticsTransmission else None)
872  filterTransmission = (dataRef.get("transmission_filter")
873  if self.config.doUseFilterTransmission else None)
874  sensorTransmission = (dataRef.get("transmission_sensor")
875  if self.config.doUseSensorTransmission else None)
876  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
877  if self.config.doUseAtmosphereTransmission else None)
878  else:
879  opticsTransmission = None
880  filterTransmission = None
881  sensorTransmission = None
882  atmosphereTransmission = None
883 
884  # Struct should include only kwargs to run()
885  return pipeBase.Struct(bias=biasExposure,
886  linearizer=linearizer,
887  crosstalkSources=crosstalkSources,
888  dark=darkExposure,
889  flat=flatExposure,
890  bfKernel=brighterFatterKernel,
891  defects=defectList,
892  fringes=fringeStruct,
893  opticsTransmission=opticsTransmission,
894  filterTransmission=filterTransmission,
895  sensorTransmission=sensorTransmission,
896  atmosphereTransmission=atmosphereTransmission,
897  )
898 
899  @pipeBase.timeMethod
900  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
901  dark=None, flat=None, bfKernel=None, defects=None, fringes=None,
902  opticsTransmission=None, filterTransmission=None,
903  sensorTransmission=None, atmosphereTransmission=None,
904  detectorNum=None, isGen3=False
905  ):
906  """!Perform instrument signature removal on an exposure.
907 
908  Steps included in the ISR processing, in order performed, are:
909  - saturation and suspect pixel masking
910  - overscan subtraction
911  - CCD assembly of individual amplifiers
912  - bias subtraction
913  - variance image construction
914  - linearization of non-linear response
915  - crosstalk masking
916  - brighter-fatter correction
917  - dark subtraction
918  - fringe correction
919  - stray light subtraction
920  - flat correction
921  - masking of known defects and camera specific features
922  - vignette calculation
923  - appending transmission curve and distortion model
924 
925  Parameters
926  ----------
927  ccdExposure : `lsst.afw.image.Exposure`
928  The raw exposure that is to be run through ISR. The
929  exposure is modified by this method.
930  camera : `lsst.afw.cameraGeom.Camera`, optional
931  The camera geometry for this exposure. Used to select the
932  distortion model appropriate for this data.
933  bias : `lsst.afw.image.Exposure`, optional
934  Bias calibration frame.
935  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
936  Functor for linearization.
937  crosstalkSources : `list`, optional
938  List of possible crosstalk sources.
939  dark : `lsst.afw.image.Exposure`, optional
940  Dark calibration frame.
941  flat : `lsst.afw.image.Exposure`, optional
942  Flat calibration frame.
943  bfKernel : `numpy.ndarray`, optional
944  Brighter-fatter kernel.
945  defects : `list`, optional
946  List of defects.
947  fringes : `lsst.pipe.base.Struct`, optional
948  Struct containing the fringe correction data, with
949  elements:
950  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
951  - ``seed``: random seed derived from the ccdExposureId for random
952  number generator (`uint32`)
953  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
954  A ``TransmissionCurve`` that represents the throughput of the optics,
955  to be evaluated in focal-plane coordinates.
956  filterTransmission : `lsst.afw.image.TransmissionCurve`
957  A ``TransmissionCurve`` that represents the throughput of the filter
958  itself, to be evaluated in focal-plane coordinates.
959  sensorTransmission : `lsst.afw.image.TransmissionCurve`
960  A ``TransmissionCurve`` that represents the throughput of the sensor
961  itself, to be evaluated in post-assembly trimmed detector coordinates.
962  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
963  A ``TransmissionCurve`` that represents the throughput of the
964  atmosphere, assumed to be spatially constant.
965  detectorNum : `int`, optional
966  The integer number for the detector to process.
967  isGen3 : bool, optional
968  Flag this call to run() as using the Gen3 butler environment.
969 
970  Returns
971  -------
972  result : `lsst.pipe.base.Struct`
973  Result struct with component:
974  - ``exposure`` : `afw.image.Exposure`
975  The fully ISR corrected exposure.
976  - ``outputExposure`` : `afw.image.Exposure`
977  An alias for `exposure`
978  - ``ossThumb`` : `numpy.ndarray`
979  Thumbnail image of the exposure after overscan subtraction.
980  - ``flattenedThumb`` : `numpy.ndarray`
981  Thumbnail image of the exposure after flat-field correction.
982 
983  Raises
984  ------
985  RuntimeError
986  Raised if a configuration option is set to True, but the
987  required calibration data has not been specified.
988 
989  Notes
990  -----
991  The current processed exposure can be viewed by setting the
992  appropriate lsstDebug entries in the `debug.display`
993  dictionary. The names of these entries correspond to some of
994  the IsrTaskConfig Boolean options, with the value denoting the
995  frame to use. The exposure is shown inside the matching
996  option check and after the processing of that step has
997  finished. The steps with debug points are:
998 
999  doAssembleCcd
1000  doBias
1001  doCrosstalk
1002  doBrighterFatter
1003  doDark
1004  doFringe
1005  doStrayLight
1006  doFlat
1007 
1008  In addition, setting the "postISRCCD" entry displays the
1009  exposure after all ISR processing has finished.
1010 
1011  """
1012 
1013  if isGen3 is True:
1014  # Gen3 currently cannot automatically do configuration overrides.
1015  # DM-15257 looks to discuss this issue.
1016 
1017  self.config.doFringe = False
1018 
1019  # Configure input exposures;
1020  if detectorNum is None:
1021  raise RuntimeError("Must supply the detectorNum if running as Gen3")
1022 
1023  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1024  bias = self.ensureExposure(bias, camera, detectorNum)
1025  dark = self.ensureExposure(dark, camera, detectorNum)
1026  flat = self.ensureExposure(flat, camera, detectorNum)
1027  else:
1028  if isinstance(ccdExposure, ButlerDataRef):
1029  return self.runDataRef(ccdExposure)
1030 
1031  ccd = ccdExposure.getDetector()
1032 
1033  if not ccd:
1034  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
1035  ccd = [FakeAmp(ccdExposure, self.config)]
1036 
1037  # Validate Input
1038  if self.config.doBias and bias is None:
1039  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1040  if self.doLinearize(ccd) and linearizer is None:
1041  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1042  if self.config.doBrighterFatter and bfKernel is None:
1043  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1044  if self.config.doDark and dark is None:
1045  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1046  if fringes is None:
1047  fringes = pipeBase.Struct(fringes=None)
1048  if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
1049  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1050  if self.config.doFlat and flat is None:
1051  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1052  if self.config.doDefect and defects is None:
1053  raise RuntimeError("Must supply defects if config.doDefect=True.")
1054  if self.config.doAddDistortionModel and camera is None:
1055  raise RuntimeError("Must supply camera if config.doAddDistortionModel=True.")
1056 
1057  # Begin ISR processing.
1058  if self.config.doConvertIntToFloat:
1059  self.log.info("Converting exposure to floating point values")
1060  ccdExposure = self.convertIntToFloat(ccdExposure)
1061 
1062  # Amplifier level processing.
1063  overscans = []
1064  for amp in ccd:
1065  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1066  if ccdExposure.getBBox().contains(amp.getBBox()):
1067  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1068  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1069 
1070  if self.config.doOverscan and not badAmp:
1071  # Overscan correction on amp-by-amp basis.
1072  overscanResults = self.overscanCorrection(ccdExposure, amp)
1073  self.log.debug("Corrected overscan for amplifier %s" % (amp.getName()))
1074  if self.config.qa is not None and self.config.qa.saveStats is True:
1075  if isinstance(overscanResults.overscanFit, float):
1076  qaMedian = overscanResults.overscanFit
1077  qaStdev = float("NaN")
1078  else:
1079  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1080  afwMath.MEDIAN | afwMath.STDEVCLIP)
1081  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1082  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1083 
1084  self.metadata.set("ISR OSCAN {} MEDIAN".format(amp.getName()), qaMedian)
1085  self.metadata.set("ISR OSCAN {} STDEV".format(amp.getName()), qaStdev)
1086  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f" %
1087  (amp.getName(), qaMedian, qaStdev))
1088  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1089  else:
1090  self.log.warn("Amplifier %s is bad." % (amp.getName()))
1091  overscanResults = None
1092 
1093  overscans.append(overscanResults if overscanResults is not None else None)
1094  else:
1095  self.log.info("Skipped OSCAN")
1096 
1097  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1098  self.log.info("Applying crosstalk correction.")
1099  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1100  self.debugView(ccdExposure, "doCrosstalk")
1101 
1102  if self.config.doAssembleCcd:
1103  self.log.info("Assembling CCD from amplifiers")
1104  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1105 
1106  if self.config.expectWcs and not ccdExposure.getWcs():
1107  self.log.warn("No WCS found in input exposure")
1108  self.debugView(ccdExposure, "doAssembleCcd")
1109 
1110  ossThumb = None
1111  if self.config.qa.doThumbnailOss:
1112  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1113 
1114  if self.config.doBias:
1115  self.log.info("Applying bias correction.")
1116  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1117  trimToFit=self.config.doTrimToMatchCalib)
1118  self.debugView(ccdExposure, "doBias")
1119 
1120  if self.config.doVariance:
1121  for amp, overscanResults in zip(ccd, overscans):
1122  if ccdExposure.getBBox().contains(amp.getBBox()):
1123  self.log.debug("Constructing variance map for amplifer %s" % (amp.getName()))
1124  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1125  if overscanResults is not None:
1126  self.updateVariance(ampExposure, amp,
1127  overscanImage=overscanResults.overscanImage)
1128  else:
1129  self.updateVariance(ampExposure, amp,
1130  overscanImage=None)
1131  if self.config.qa is not None and self.config.qa.saveStats is True:
1132  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1133  afwMath.MEDIAN | afwMath.STDEVCLIP)
1134  self.metadata.set("ISR VARIANCE {} MEDIAN".format(amp.getName()),
1135  qaStats.getValue(afwMath.MEDIAN))
1136  self.metadata.set("ISR VARIANCE {} STDEV".format(amp.getName()),
1137  qaStats.getValue(afwMath.STDEVCLIP))
1138  self.log.debug(" Variance stats for amplifer %s: %f +/- %f" %
1139  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1140  qaStats.getValue(afwMath.STDEVCLIP)))
1141 
1142  if self.doLinearize(ccd):
1143  self.log.info("Applying linearizer.")
1144  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1145 
1146  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1147  self.log.info("Applying crosstalk correction.")
1148  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1149  self.debugView(ccdExposure, "doCrosstalk")
1150 
1151  if self.config.doWidenSaturationTrails:
1152  self.log.info("Widening saturation trails.")
1153  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1154 
1155  interpolationDone = False
1156  if self.config.doBrighterFatter:
1157  # We need to apply flats and darks before we can interpolate, and we
1158  # need to interpolate before we do B-F, but we do B-F without the
1159  # flats and darks applied so we can work in units of electrons or holes.
1160  # This context manager applies and then removes the darks and flats.
1161  with self.flatContext(ccdExposure, flat, dark):
1162  if self.config.doDefect:
1163  self.maskAndInterpDefect(ccdExposure, defects)
1164 
1165  if self.config.doSaturationInterpolation:
1166  self.saturationInterpolation(ccdExposure)
1167 
1168  self.maskAndInterpNan(ccdExposure)
1169  interpolationDone = True
1170 
1171  if self.config.brighterFatterLevel == 'DETECTOR':
1172  kernelElement = bfKernel
1173  else:
1174  # TODO: DM-15631 for implementing this
1175  raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented")
1176  self.log.info("Applying brighter fatter correction.")
1177  isrFunctions.brighterFatterCorrection(ccdExposure, kernelElement,
1178  self.config.brighterFatterMaxIter,
1179  self.config.brighterFatterThreshold,
1180  self.config.brighterFatterApplyGain,
1181  )
1182  self.debugView(ccdExposure, "doBrighterFatter")
1183 
1184  if self.config.doDark:
1185  self.log.info("Applying dark correction.")
1186  self.darkCorrection(ccdExposure, dark)
1187  self.debugView(ccdExposure, "doDark")
1188 
1189  if self.config.doFringe and not self.config.fringeAfterFlat:
1190  self.log.info("Applying fringe correction before flat.")
1191  self.fringe.run(ccdExposure, **fringes.getDict())
1192  self.debugView(ccdExposure, "doFringe")
1193 
1194  if self.config.doStrayLight:
1195  self.log.info("Applying stray light correction.")
1196  self.strayLight.run(ccdExposure)
1197  self.debugView(ccdExposure, "doStrayLight")
1198 
1199  if self.config.doFlat:
1200  self.log.info("Applying flat correction.")
1201  self.flatCorrection(ccdExposure, flat)
1202  self.debugView(ccdExposure, "doFlat")
1203 
1204  if self.config.doApplyGains:
1205  self.log.info("Applying gain correction instead of flat.")
1206  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1207 
1208  if self.config.doDefect and not interpolationDone:
1209  self.log.info("Masking and interpolating defects.")
1210  self.maskAndInterpDefect(ccdExposure, defects)
1211 
1212  if self.config.doSaturation and not interpolationDone:
1213  self.log.info("Interpolating saturated pixels.")
1214  self.saturationInterpolation(ccdExposure)
1215 
1216  if self.config.doNanInterpAfterFlat or not interpolationDone:
1217  self.log.info("Masking and interpolating NAN value pixels.")
1218  self.maskAndInterpNan(ccdExposure)
1219 
1220  if self.config.doFringe and self.config.fringeAfterFlat:
1221  self.log.info("Applying fringe correction after flat.")
1222  self.fringe.run(ccdExposure, **fringes.getDict())
1223 
1224  if self.config.doSetBadRegions:
1225  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1226  self.log.info("Set %d BAD pixels to %f." % (badPixelCount, badPixelValue))
1227 
1228  flattenedThumb = None
1229  if self.config.qa.doThumbnailFlattened:
1230  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1231 
1232  if self.config.doCameraSpecificMasking:
1233  self.log.info("Masking regions for camera specific reasons.")
1234  self.masking.run(ccdExposure)
1235 
1236  self.roughZeroPoint(ccdExposure)
1237 
1238  if self.config.doVignette:
1239  self.log.info("Constructing Vignette polygon.")
1240  self.vignettePolygon = self.vignette.run(ccdExposure)
1241 
1242  if self.config.vignette.doWriteVignettePolygon:
1243  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1244 
1245  if self.config.doAttachTransmissionCurve:
1246  self.log.info("Adding transmission curves.")
1247  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1248  filterTransmission=filterTransmission,
1249  sensorTransmission=sensorTransmission,
1250  atmosphereTransmission=atmosphereTransmission)
1251 
1252  if self.config.doAddDistortionModel:
1253  self.log.info("Adding a distortion model to the WCS.")
1254  isrFunctions.addDistortionModel(exposure=ccdExposure, camera=camera)
1255 
1256  if self.config.doMeasureBackground:
1257  self.log.info("Measuring background level:")
1258  self.measureBackground(ccdExposure, self.config.qa)
1259 
1260  if self.config.qa is not None and self.config.qa.saveStats is True:
1261  for amp in ccd:
1262  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1263  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1264  afwMath.MEDIAN | afwMath.STDEVCLIP)
1265  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1266  qaStats.getValue(afwMath.MEDIAN))
1267  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1268  qaStats.getValue(afwMath.STDEVCLIP))
1269  self.log.debug(" Background stats for amplifer %s: %f +/- %f" %
1270  (amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1271  qaStats.getValue(afwMath.STDEVCLIP)))
1272 
1273  self.debugView(ccdExposure, "postISRCCD")
1274 
1275  return pipeBase.Struct(
1276  exposure=ccdExposure,
1277  ossThumb=ossThumb,
1278  flattenedThumb=flattenedThumb,
1279 
1280  outputExposure=ccdExposure,
1281  outputOssThumbnail=ossThumb,
1282  outputFlattenedThumbnail=flattenedThumb,
1283  )
1284 
1285  @pipeBase.timeMethod
1286  def runDataRef(self, sensorRef):
1287  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1288 
1289  This method contains the `CmdLineTask` interface to the ISR
1290  processing. All IO is handled here, freeing the `run()` method
1291  to manage only pixel-level calculations. The steps performed
1292  are:
1293  - Read in necessary detrending/isr/calibration data.
1294  - Process raw exposure in `run()`.
1295  - Persist the ISR-corrected exposure as "postISRCCD" if
1296  config.doWrite=True.
1297 
1298  Parameters
1299  ----------
1300  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1301  DataRef of the detector data to be processed
1302 
1303  Returns
1304  -------
1305  result : `lsst.pipe.base.Struct`
1306  Result struct with component:
1307  - ``exposure`` : `afw.image.Exposure`
1308  The fully ISR corrected exposure.
1309 
1310  Raises
1311  ------
1312  RuntimeError
1313  Raised if a configuration option is set to True, but the
1314  required calibration data does not exist.
1315 
1316  """
1317  self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId))
1318 
1319  ccdExposure = sensorRef.get(self.config.datasetType)
1320 
1321  camera = sensorRef.get("camera")
1322  if camera is None and self.config.doAddDistortionModel:
1323  raise RuntimeError("config.doAddDistortionModel is True "
1324  "but could not get a camera from the butler")
1325  isrData = self.readIsrData(sensorRef, ccdExposure)
1326 
1327  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1328 
1329  if self.config.doWrite:
1330  sensorRef.put(result.exposure, "postISRCCD")
1331  if result.ossThumb is not None:
1332  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1333  if result.flattenedThumb is not None:
1334  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1335 
1336  return result
1337 
1338  def getIsrExposure(self, dataRef, datasetType, immediate=True):
1339  """!Retrieve a calibration dataset for removing instrument signature.
1340 
1341  Parameters
1342  ----------
1343 
1344  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1345  DataRef of the detector data to find calibration datasets
1346  for.
1347  datasetType : `str`
1348  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1349  immediate : `Bool`
1350  If True, disable butler proxies to enable error handling
1351  within this routine.
1352 
1353  Returns
1354  -------
1355  exposure : `lsst.afw.image.Exposure`
1356  Requested calibration frame.
1357 
1358  Raises
1359  ------
1360  RuntimeError
1361  Raised if no matching calibration frame can be found.
1362  """
1363  try:
1364  exp = dataRef.get(datasetType, immediate=immediate)
1365  except Exception as exc1:
1366  if not self.config.fallbackFilterName:
1367  raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1))
1368  try:
1369  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1370  except Exception as exc2:
1371  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" %
1372  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1373  self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName)
1374 
1375  if self.config.doAssembleIsrExposures:
1376  exp = self.assembleCcd.assembleCcd(exp)
1377  return exp
1378 
1379  def ensureExposure(self, inputExp, camera, detectorNum):
1380  """Ensure that the data returned by Butler is a fully constructed exposure.
1381 
1382  ISR requires exposure-level image data for historical reasons, so if we did
1383  not recieve that from Butler, construct it from what we have, modifying the
1384  input in place.
1385 
1386  Parameters
1387  ----------
1388  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1389  `lsst.afw.image.ImageF`
1390  The input data structure obtained from Butler.
1391  camera : `lsst.afw.cameraGeom.camera`
1392  The camera associated with the image. Used to find the appropriate
1393  detector.
1394  detectorNum : `int`
1395  The detector this exposure should match.
1396 
1397  Returns
1398  -------
1399  inputExp : `lsst.afw.image.Exposure`
1400  The re-constructed exposure, with appropriate detector parameters.
1401 
1402  Raises
1403  ------
1404  TypeError
1405  Raised if the input data cannot be used to construct an exposure.
1406  """
1407  if isinstance(inputExp, afwImage.DecoratedImageU):
1408  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1409  elif isinstance(inputExp, afwImage.ImageF):
1410  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1411  elif isinstance(inputExp, afwImage.MaskedImageF):
1412  inputExp = afwImage.makeExposure(inputExp)
1413  elif isinstance(inputExp, afwImage.Exposure):
1414  pass
1415  else:
1416  raise TypeError(f"Input Exposure is not known type in isrTask.ensureExposure: {type(inputExp)}")
1417 
1418  if inputExp.getDetector() is None:
1419  inputExp.setDetector(camera[detectorNum])
1420 
1421  return inputExp
1422 
1423  def convertIntToFloat(self, exposure):
1424  """Convert exposure image from uint16 to float.
1425 
1426  If the exposure does not need to be converted, the input is
1427  immediately returned. For exposures that are converted to use
1428  floating point pixels, the variance is set to unity and the
1429  mask to zero.
1430 
1431  Parameters
1432  ----------
1433  exposure : `lsst.afw.image.Exposure`
1434  The raw exposure to be converted.
1435 
1436  Returns
1437  -------
1438  newexposure : `lsst.afw.image.Exposure`
1439  The input ``exposure``, converted to floating point pixels.
1440 
1441  Raises
1442  ------
1443  RuntimeError
1444  Raised if the exposure type cannot be converted to float.
1445 
1446  """
1447  if isinstance(exposure, afwImage.ExposureF):
1448  # Nothing to be done
1449  return exposure
1450  if not hasattr(exposure, "convertF"):
1451  raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure))
1452 
1453  newexposure = exposure.convertF()
1454  newexposure.variance[:] = 1
1455  newexposure.mask[:] = 0x0
1456 
1457  return newexposure
1458 
1459  def maskAmplifier(self, ccdExposure, amp, defects):
1460  """Identify bad amplifiers, saturated and suspect pixels.
1461 
1462  Parameters
1463  ----------
1464  ccdExposure : `lsst.afw.image.Exposure`
1465  Input exposure to be masked.
1466  amp : `lsst.afw.table.AmpInfoCatalog`
1467  Catalog of parameters defining the amplifier on this
1468  exposure to mask.
1469  defects : `list`
1470  List of defects. Used to determine if the entire
1471  amplifier is bad.
1472 
1473  Returns
1474  -------
1475  badAmp : `Bool`
1476  If this is true, the entire amplifier area is covered by
1477  defects and unusable.
1478 
1479  """
1480  maskedImage = ccdExposure.getMaskedImage()
1481 
1482  badAmp = False
1483 
1484  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1485  # comparison with current defects definition.
1486  if defects is not None:
1487  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1488 
1489  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1490  # association with pixels in current ccdExposure).
1491  if badAmp:
1492  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1493  afwImage.PARENT)
1494  maskView = dataView.getMask()
1495  maskView |= maskView.getPlaneBitMask("BAD")
1496  del maskView
1497  return badAmp
1498 
1499  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1500  # Saturation and suspect pixels can be masked now, though.
1501  limits = dict()
1502  if self.config.doSaturation and not badAmp:
1503  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1504  if self.config.doSuspect and not badAmp:
1505  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1506 
1507  for maskName, maskThreshold in limits.items():
1508  if not math.isnan(maskThreshold):
1509  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1510  isrFunctions.makeThresholdMask(
1511  maskedImage=dataView,
1512  threshold=maskThreshold,
1513  growFootprints=0,
1514  maskName=maskName
1515  )
1516 
1517  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1518  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1519  afwImage.PARENT)
1520  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1521  self.config.suspectMaskName])
1522  if numpy.all(maskView.getArray() & maskVal > 0):
1523  badAmp = True
1524 
1525  return badAmp
1526 
1527  def overscanCorrection(self, ccdExposure, amp):
1528  """Apply overscan correction in place.
1529 
1530  This method does initial pixel rejection of the overscan
1531  region. The overscan can also be optionally segmented to
1532  allow for discontinuous overscan responses to be fit
1533  separately. The actual overscan subtraction is performed by
1534  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1535  which is called here after the amplifier is preprocessed.
1536 
1537  Parameters
1538  ----------
1539  ccdExposure : `lsst.afw.image.Exposure`
1540  Exposure to have overscan correction performed.
1541  amp : `lsst.afw.table.AmpInfoCatalog`
1542  The amplifier to consider while correcting the overscan.
1543 
1544  Returns
1545  -------
1546  overscanResults : `lsst.pipe.base.Struct`
1547  Result struct with components:
1548  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1549  Value or fit subtracted from the amplifier image data.
1550  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1551  Value or fit subtracted from the overscan image data.
1552  - ``overscanImage`` : `lsst.afw.image.Image`
1553  Image of the overscan region with the overscan
1554  correction applied. This quantity is used to estimate
1555  the amplifier read noise empirically.
1556 
1557  Raises
1558  ------
1559  RuntimeError
1560  Raised if the ``amp`` does not contain raw pixel information.
1561 
1562  See Also
1563  --------
1564  lsst.ip.isr.isrFunctions.overscanCorrection
1565  """
1566  if not amp.getHasRawInfo():
1567  raise RuntimeError("This method must be executed on an amp with raw information.")
1568 
1569  if amp.getRawHorizontalOverscanBBox().isEmpty():
1570  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1571  return None
1572 
1573  # Construct views
1574  ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(),
1575  afwImage.PARENT)
1576  overscanImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
1577  amp.getRawHorizontalOverscanBBox(),
1578  afwImage.PARENT)
1579  overscanArray = overscanImage.getImage().getArray()
1580 
1581  statControl = afwMath.StatisticsControl()
1582  statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT"))
1583 
1584  # Determine the bounding boxes
1585  dataBBox = amp.getRawDataBBox()
1586  oscanBBox = amp.getRawHorizontalOverscanBBox()
1587  x0 = 0
1588  x1 = 0
1589 
1590  prescanBBox = amp.getRawPrescanBBox()
1591  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1592  x0 += self.config.overscanNumLeadingColumnsToSkip
1593  x1 -= self.config.overscanNumTrailingColumnsToSkip
1594  else:
1595  x0 += self.config.overscanNumTrailingColumnsToSkip
1596  x1 -= self.config.overscanNumLeadingColumnsToSkip
1597 
1598  # Determine if we need to work on subregions of the amplifier and overscan.
1599  imageBBoxes = []
1600  overscanBBoxes = []
1601 
1602  if ((self.config.overscanBiasJump and
1603  self.config.overscanBiasJumpLocation) and
1604  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1605  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1606  self.config.overscanBiasJumpDevices)):
1607  if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
1608  yLower = self.config.overscanBiasJumpLocation
1609  yUpper = dataBBox.getHeight() - yLower
1610  else:
1611  yUpper = self.config.overscanBiasJumpLocation
1612  yLower = dataBBox.getHeight() - yUpper
1613 
1614  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1615  afwGeom.Extent2I(dataBBox.getWidth(), yLower)))
1616  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() +
1617  afwGeom.Extent2I(x0, 0),
1618  afwGeom.Extent2I(oscanBBox.getWidth() + x1, yLower)))
1619 
1620  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin() + afwGeom.Extent2I(0, yLower),
1621  afwGeom.Extent2I(dataBBox.getWidth(), yUpper)))
1622 
1623  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(x0, yLower),
1624  afwGeom.Extent2I(oscanBBox.getWidth() + x1, yUpper)))
1625  else:
1626  imageBBoxes.append(afwGeom.Box2I(dataBBox.getBegin(),
1627  afwGeom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1628 
1629  overscanBBoxes.append(afwGeom.Box2I(oscanBBox.getBegin() + afwGeom.Extent2I(x0, 0),
1630  afwGeom.Extent2I(oscanBBox.getWidth() + x1,
1631  oscanBBox.getHeight())))
1632 
1633  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1634  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1635  ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), imageBBox,
1636  afwImage.PARENT)
1637  overscanImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), overscanBBox,
1638  afwImage.PARENT)
1639 
1640  overscanArray = overscanImage.getImage().getArray()
1641  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.getMask().getArray(),
1642  overscanArray))
1643  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1644  overscanImage.getMask().getArray()[bad] = overscanImage.getMask().getPlaneBitMask("SAT")
1645 
1646  statControl = afwMath.StatisticsControl()
1647  statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT"))
1648 
1649  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1650  overscanImage=overscanImage,
1651  fitType=self.config.overscanFitType,
1652  order=self.config.overscanOrder,
1653  collapseRej=self.config.overscanNumSigmaClip,
1654  statControl=statControl,
1655  overscanIsInt=self.config.overscanIsInt
1656  )
1657 
1658  # Measure average overscan levels and record them in the metadata
1659  levelStat = afwMath.MEDIAN
1660  sigmaStat = afwMath.STDEVCLIP
1661 
1662  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1663  self.config.qa.flatness.nIter)
1664  metadata = ccdExposure.getMetadata()
1665  ampNum = amp.getName()
1666  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1667  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1668  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1669  else:
1670  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1671  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1672  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1673 
1674  return overscanResults
1675 
1676  def updateVariance(self, ampExposure, amp, overscanImage=None):
1677  """Set the variance plane using the amplifier gain and read noise
1678 
1679  The read noise is calculated from the ``overscanImage`` if the
1680  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1681  the value from the amplifier data is used.
1682 
1683  Parameters
1684  ----------
1685  ampExposure : `lsst.afw.image.Exposure`
1686  Exposure to process.
1687  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1688  Amplifier detector data.
1689  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1690  Image of overscan, required only for empirical read noise.
1691 
1692  See also
1693  --------
1694  lsst.ip.isr.isrFunctions.updateVariance
1695  """
1696  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1697  gain = amp.getGain()
1698 
1699  if math.isnan(gain):
1700  gain = 1.0
1701  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1702  elif gain <= 0:
1703  patchedGain = 1.0
1704  self.log.warn("Gain for amp %s == %g <= 0; setting to %f" %
1705  (amp.getName(), gain, patchedGain))
1706  gain = patchedGain
1707 
1708  if self.config.doEmpiricalReadNoise and overscanImage is None:
1709  self.log.info("Overscan is none for EmpiricalReadNoise")
1710 
1711  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1712  stats = afwMath.StatisticsControl()
1713  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1714  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1715  self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise)
1716  else:
1717  readNoise = amp.getReadNoise()
1718 
1719  isrFunctions.updateVariance(
1720  maskedImage=ampExposure.getMaskedImage(),
1721  gain=gain,
1722  readNoise=readNoise,
1723  )
1724 
1725  def darkCorrection(self, exposure, darkExposure, invert=False):
1726  """!Apply dark correction in place.
1727 
1728  Parameters
1729  ----------
1730  exposure : `lsst.afw.image.Exposure`
1731  Exposure to process.
1732  darkExposure : `lsst.afw.image.Exposure`
1733  Dark exposure of the same size as ``exposure``.
1734  invert : `Bool`, optional
1735  If True, re-add the dark to an already corrected image.
1736 
1737  Raises
1738  ------
1739  RuntimeError
1740  Raised if either ``exposure`` or ``darkExposure`` do not
1741  have their dark time defined.
1742 
1743  See Also
1744  --------
1745  lsst.ip.isr.isrFunctions.darkCorrection
1746  """
1747  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1748  if math.isnan(expScale):
1749  raise RuntimeError("Exposure darktime is NAN")
1750  if darkExposure.getInfo().getVisitInfo() is not None:
1751  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1752  else:
1753  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1754  # so getDarkTime() does not exist.
1755  darkScale = 1.0
1756 
1757  if math.isnan(darkScale):
1758  raise RuntimeError("Dark calib darktime is NAN")
1759  isrFunctions.darkCorrection(
1760  maskedImage=exposure.getMaskedImage(),
1761  darkMaskedImage=darkExposure.getMaskedImage(),
1762  expScale=expScale,
1763  darkScale=darkScale,
1764  invert=invert,
1765  trimToFit=self.config.doTrimToMatchCalib
1766  )
1767 
1768  def doLinearize(self, detector):
1769  """!Check if linearization is needed for the detector cameraGeom.
1770 
1771  Checks config.doLinearize and the linearity type of the first
1772  amplifier.
1773 
1774  Parameters
1775  ----------
1776  detector : `lsst.afw.cameraGeom.Detector`
1777  Detector to get linearity type from.
1778 
1779  Returns
1780  -------
1781  doLinearize : `Bool`
1782  If True, linearization should be performed.
1783  """
1784  return self.config.doLinearize and \
1785  detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType
1786 
1787  def flatCorrection(self, exposure, flatExposure, invert=False):
1788  """!Apply flat correction in place.
1789 
1790  Parameters
1791  ----------
1792  exposure : `lsst.afw.image.Exposure`
1793  Exposure to process.
1794  flatExposure : `lsst.afw.image.Exposure`
1795  Flat exposure of the same size as ``exposure``.
1796  invert : `Bool`, optional
1797  If True, unflatten an already flattened image.
1798 
1799  See Also
1800  --------
1801  lsst.ip.isr.isrFunctions.flatCorrection
1802  """
1803  isrFunctions.flatCorrection(
1804  maskedImage=exposure.getMaskedImage(),
1805  flatMaskedImage=flatExposure.getMaskedImage(),
1806  scalingType=self.config.flatScalingType,
1807  userScale=self.config.flatUserScale,
1808  invert=invert,
1809  trimToFit=self.config.doTrimToMatchCalib
1810  )
1811 
1812  def saturationDetection(self, exposure, amp):
1813  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
1814 
1815  Parameters
1816  ----------
1817  exposure : `lsst.afw.image.Exposure`
1818  Exposure to process. Only the amplifier DataSec is processed.
1819  amp : `lsst.afw.table.AmpInfoCatalog`
1820  Amplifier detector data.
1821 
1822  See Also
1823  --------
1824  lsst.ip.isr.isrFunctions.makeThresholdMask
1825  """
1826  if not math.isnan(amp.getSaturation()):
1827  maskedImage = exposure.getMaskedImage()
1828  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1829  isrFunctions.makeThresholdMask(
1830  maskedImage=dataView,
1831  threshold=amp.getSaturation(),
1832  growFootprints=0,
1833  maskName=self.config.saturatedMaskName,
1834  )
1835 
1836  def saturationInterpolation(self, ccdExposure):
1837  """!Interpolate over saturated pixels, in place.
1838 
1839  This method should be called after `saturationDetection`, to
1840  ensure that the saturated pixels have been identified in the
1841  SAT mask. It should also be called after `assembleCcd`, since
1842  saturated regions may cross amplifier boundaries.
1843 
1844  Parameters
1845  ----------
1846  exposure : `lsst.afw.image.Exposure`
1847  Exposure to process.
1848 
1849  See Also
1850  --------
1851  lsst.ip.isr.isrTask.saturationDetection
1852  lsst.ip.isr.isrFunctions.interpolateFromMask
1853  """
1854  isrFunctions.interpolateFromMask(
1855  maskedImage=ccdExposure.getMaskedImage(),
1856  fwhm=self.config.fwhm,
1857  growFootprints=self.config.growSaturationFootprintSize,
1858  maskName=self.config.saturatedMaskName,
1859  )
1860 
1861  def suspectDetection(self, exposure, amp):
1862  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
1863 
1864  Parameters
1865  ----------
1866  exposure : `lsst.afw.image.Exposure`
1867  Exposure to process. Only the amplifier DataSec is processed.
1868  amp : `lsst.afw.table.AmpInfoCatalog`
1869  Amplifier detector data.
1870 
1871  See Also
1872  --------
1873  lsst.ip.isr.isrFunctions.makeThresholdMask
1874 
1875  Notes
1876  -----
1877  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
1878  This is intended to indicate pixels that may be affected by unknown systematics;
1879  for example if non-linearity corrections above a certain level are unstable
1880  then that would be a useful value for suspectLevel. A value of `nan` indicates
1881  that no such level exists and no pixels are to be masked as suspicious.
1882  """
1883  suspectLevel = amp.getSuspectLevel()
1884  if math.isnan(suspectLevel):
1885  return
1886 
1887  maskedImage = exposure.getMaskedImage()
1888  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1889  isrFunctions.makeThresholdMask(
1890  maskedImage=dataView,
1891  threshold=suspectLevel,
1892  growFootprints=0,
1893  maskName=self.config.suspectMaskName,
1894  )
1895 
1896  def maskAndInterpDefect(self, ccdExposure, defectBaseList):
1897  """!Mask defects using mask plane "BAD" and interpolate over them, in place.
1898 
1899  Parameters
1900  ----------
1901  ccdExposure : `lsst.afw.image.Exposure`
1902  Exposure to process.
1903  defectBaseList : `List`
1904  List of defects to mask and interpolate.
1905 
1906  Notes
1907  -----
1908  Call this after CCD assembly, since defects may cross amplifier boundaries.
1909  """
1910  maskedImage = ccdExposure.getMaskedImage()
1911  defectList = []
1912  for d in defectBaseList:
1913  bbox = d.getBBox()
1914  nd = measAlg.Defect(bbox)
1915  defectList.append(nd)
1916  isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')
1917  isrFunctions.interpolateDefectList(
1918  maskedImage=maskedImage,
1919  defectList=defectList,
1920  fwhm=self.config.fwhm,
1921  )
1922 
1923  if self.config.numEdgeSuspect > 0:
1924  goodBBox = maskedImage.getBBox()
1925  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
1926  goodBBox.grow(-self.config.numEdgeSuspect)
1927  # Mask pixels outside goodBBox as SUSPECT
1928  SourceDetectionTask.setEdgeBits(
1929  maskedImage,
1930  goodBBox,
1931  maskedImage.getMask().getPlaneBitMask("SUSPECT")
1932  )
1933 
1934  def maskAndInterpNan(self, exposure):
1935  """!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
1936 
1937  Parameters
1938  ----------
1939  exposure : `lsst.afw.image.Exposure`
1940  Exposure to process.
1941 
1942  Notes
1943  -----
1944  We mask and interpolate over all NaNs, including those
1945  that are masked with other bits (because those may or may
1946  not be interpolated over later, and we want to remove all
1947  NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane
1948  is used to preserve the historical name.
1949  """
1950  maskedImage = exposure.getMaskedImage()
1951 
1952  # Find and mask NaNs
1953  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
1954  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
1955  numNans = maskNans(maskedImage, maskVal)
1956  self.metadata.set("NUMNANS", numNans)
1957 
1958  # Interpolate over these previously-unmasked NaNs
1959  if numNans > 0:
1960  self.log.warn("There were %i unmasked NaNs", numNans)
1961  nanDefectList = isrFunctions.getDefectListFromMask(
1962  maskedImage=maskedImage,
1963  maskName='UNMASKEDNAN',
1964  )
1965  isrFunctions.interpolateDefectList(
1966  maskedImage=exposure.getMaskedImage(),
1967  defectList=nanDefectList,
1968  fwhm=self.config.fwhm,
1969  )
1970 
1971  def measureBackground(self, exposure, IsrQaConfig=None):
1972  """Measure the image background in subgrids, for quality control purposes.
1973 
1974  Parameters
1975  ----------
1976  exposure : `lsst.afw.image.Exposure`
1977  Exposure to process.
1978  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
1979  Configuration object containing parameters on which background
1980  statistics and subgrids to use.
1981  """
1982  if IsrQaConfig is not None:
1983  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
1984  IsrQaConfig.flatness.nIter)
1985  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
1986  statsControl.setAndMask(maskVal)
1987  maskedImage = exposure.getMaskedImage()
1988  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
1989  skyLevel = stats.getValue(afwMath.MEDIAN)
1990  skySigma = stats.getValue(afwMath.STDEVCLIP)
1991  self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
1992  metadata = exposure.getMetadata()
1993  metadata.set('SKYLEVEL', skyLevel)
1994  metadata.set('SKYSIGMA', skySigma)
1995 
1996  # calcluating flatlevel over the subgrids
1997  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
1998  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
1999  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2000  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2001  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2002  skyLevels = numpy.zeros((nX, nY))
2003 
2004  for j in range(nY):
2005  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2006  for i in range(nX):
2007  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2008 
2009  xLLC = xc - meshXHalf
2010  yLLC = yc - meshYHalf
2011  xURC = xc + meshXHalf - 1
2012  yURC = yc + meshYHalf - 1
2013 
2014  bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
2015  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2016 
2017  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2018 
2019  good = numpy.where(numpy.isfinite(skyLevels))
2020  skyMedian = numpy.median(skyLevels[good])
2021  flatness = (skyLevels[good] - skyMedian) / skyMedian
2022  flatness_rms = numpy.std(flatness)
2023  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2024 
2025  self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
2026  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" %
2027  (nX, nY, flatness_pp, flatness_rms))
2028 
2029  metadata.set('FLATNESS_PP', float(flatness_pp))
2030  metadata.set('FLATNESS_RMS', float(flatness_rms))
2031  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2032  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2033  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2034 
2035  def roughZeroPoint(self, exposure):
2036  """Set an approximate magnitude zero point for the exposure.
2037 
2038  Parameters
2039  ----------
2040  exposure : `lsst.afw.image.Exposure`
2041  Exposure to process.
2042  """
2043  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2044  if filterName in self.config.fluxMag0T1:
2045  fluxMag0 = self.config.fluxMag0T1[filterName]
2046  else:
2047  self.log.warn("No rough magnitude zero point set for filter %s" % filterName)
2048  fluxMag0 = self.config.defaultFluxMag0T1
2049 
2050  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2051  if not expTime > 0: # handle NaN as well as <= 0
2052  self.log.warn("Non-positive exposure time; skipping rough zero point")
2053  return
2054 
2055  self.log.info("Setting rough magnitude zero point: %f" % (2.5*math.log10(fluxMag0*expTime),))
2056  exposure.getCalib().setFluxMag0(fluxMag0*expTime)
2057 
2058  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2059  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2060 
2061  Parameters
2062  ----------
2063  ccdExposure : `lsst.afw.image.Exposure`
2064  Exposure to process.
2065  fpPolygon : `lsst.afw.geom.Polygon`
2066  Polygon in focal plane coordinates.
2067  """
2068  # Get ccd corners in focal plane coordinates
2069  ccd = ccdExposure.getDetector()
2070  fpCorners = ccd.getCorners(FOCAL_PLANE)
2071  ccdPolygon = Polygon(fpCorners)
2072 
2073  # Get intersection of ccd corners with fpPolygon
2074  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2075 
2076  # Transform back to pixel positions and build new polygon
2077  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2078  validPolygon = Polygon(ccdPoints)
2079  ccdExposure.getInfo().setValidPolygon(validPolygon)
2080 
2081  @contextmanager
2082  def flatContext(self, exp, flat, dark=None):
2083  """Context manager that applies and removes flats and darks,
2084  if the task is configured to apply them.
2085 
2086  Parameters
2087  ----------
2088  exp : `lsst.afw.image.Exposure`
2089  Exposure to process.
2090  flat : `lsst.afw.image.Exposure`
2091  Flat exposure the same size as ``exp``.
2092  dark : `lsst.afw.image.Exposure`, optional
2093  Dark exposure the same size as ``exp``.
2094 
2095  Yields
2096  ------
2097  exp : `lsst.afw.image.Exposure`
2098  The flat and dark corrected exposure.
2099  """
2100  if self.config.doDark and dark is not None:
2101  self.darkCorrection(exp, dark)
2102  if self.config.doFlat:
2103  self.flatCorrection(exp, flat)
2104  try:
2105  yield exp
2106  finally:
2107  if self.config.doFlat:
2108  self.flatCorrection(exp, flat, invert=True)
2109  if self.config.doDark and dark is not None:
2110  self.darkCorrection(exp, dark, invert=True)
2111 
2112  def debugView(self, exposure, stepname):
2113  """Utility function to examine ISR exposure at different stages.
2114 
2115  Parameters
2116  ----------
2117  exposure : `lsst.afw.image.Exposure`
2118  Exposure to view.
2119  stepname : `str`
2120  State of processing to view.
2121  """
2122  frame = getDebugFrame(self._display, stepname)
2123  if frame:
2124  display = getDisplay(frame)
2125  display.scale('asinh', 'zscale')
2126  display.mtv(exposure)
2127 
2128 
2129 class FakeAmp(object):
2130  """A Detector-like object that supports returning gain and saturation level
2131 
2132  This is used when the input exposure does not have a detector.
2133 
2134  Parameters
2135  ----------
2136  exposure : `lsst.afw.image.Exposure`
2137  Exposure to generate a fake amplifier for.
2138  config : `lsst.ip.isr.isrTaskConfig`
2139  Configuration to apply to the fake amplifier.
2140  """
2141 
2142  def __init__(self, exposure, config):
2143  self._bbox = exposure.getBBox(afwImage.LOCAL)
2144  self._RawHorizontalOverscanBBox = afwGeom.Box2I()
2145  self._gain = config.gain
2146  self._readNoise = config.readNoise
2147  self._saturation = config.saturation
2148 
2149  def getBBox(self):
2150  return self._bbox
2151 
2152  def getRawBBox(self):
2153  return self._bbox
2154 
2155  def getHasRawInfo(self):
2156  return True # but see getRawHorizontalOverscanBBox()
2157 
2159  return self._RawHorizontalOverscanBBox
2160 
2161  def getGain(self):
2162  return self._gain
2163 
2164  def getReadNoise(self):
2165  return self._readNoise
2166 
2167  def getSaturation(self):
2168  return self._saturation
2169 
2170  def getSuspectLevel(self):
2171  return float("NaN")
2172 
2173 
2174 class RunIsrConfig(pexConfig.Config):
2175  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2176 
2177 
2178 class RunIsrTask(pipeBase.CmdLineTask):
2179  """Task to wrap the default IsrTask to allow it to be retargeted.
2180 
2181  The standard IsrTask can be called directly from a command line
2182  program, but doing so removes the ability of the task to be
2183  retargeted. As most cameras override some set of the IsrTask
2184  methods, this would remove those data-specific methods in the
2185  output post-ISR images. This wrapping class fixes the issue,
2186  allowing identical post-ISR images to be generated by both the
2187  processCcd and isrTask code.
2188  """
2189  ConfigClass = RunIsrConfig
2190  _DefaultName = "runIsr"
2191 
2192  def __init__(self, *args, **kwargs):
2193  super().__init__(*args, **kwargs)
2194  self.makeSubtask("isr")
2195 
2196  def runDataRef(self, dataRef):
2197  """
2198  Parameters
2199  ----------
2200  dataRef : `lsst.daf.persistence.ButlerDataRef`
2201  data reference of the detector data to be processed
2202 
2203  Returns
2204  -------
2205  result : `pipeBase.Struct`
2206  Result struct with component:
2207 
2208  - exposure : `lsst.afw.image.Exposure`
2209  Post-ISR processed exposure.
2210  """
2211  return self.isr.runDataRef(dataRef)
def getInputDatasetTypes(cls, config)
Definition: isrTask.py:706
def runDataRef(self, sensorRef)
Definition: isrTask.py:1286
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:1971
def debugView(self, exposure, stepname)
Definition: isrTask.py:2112
def __init__(self, kwargs)
Definition: isrTask.py:696
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1379
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:801
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: isrTask.py:755
def runDataRef(self, dataRef)
Definition: isrTask.py:2196
def __init__(self, args, kwargs)
Definition: isrTask.py:2192
def maskAndInterpNan(self, exposure)
Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place.
Definition: isrTask.py:1934
def saturationInterpolation(self, ccdExposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:1836
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2035
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2158
def getOutputDatasetTypes(cls, config)
Definition: isrTask.py:743
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1527
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1423
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:1787
def makeDatasetType(self, dsConfig)
Definition: isrTask.py:798
def getIsrExposure(self, dataRef, datasetType, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1338
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1725
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1768
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, defects=None, fringes=None, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:905
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2058
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1459
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2082
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:34
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1676
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:1861
def maskAndInterpDefect(self, ccdExposure, defectBaseList)
Mask defects using mask plane "BAD" and interpolate over them, in place.
Definition: isrTask.py:1896
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:1812
def __init__(self, exposure, config)
Definition: isrTask.py:2142