lsst.ip.isr  22.0.1-15-ge634287+36592fbd19
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 from .defects import Defects
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  findFirst=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  minimum=0, # can fall back to cameraGeom
123  )
124  # TODO: DM-25348. This does not work yet to correctly load
125  # possible crosstalk sources.
126  crosstalkSources = cT.PrerequisiteInput(
127  name="isrOverscanCorrected",
128  doc="Overscan corrected input images.",
129  storageClass="Exposure",
130  dimensions=["instrument", "exposure", "detector"],
131  deferLoad=True,
132  multiple=True,
133  lookupFunction=crosstalkSourceLookup,
134  minimum=0, # not needed for all instruments, no config to control this
135  )
136  bias = cT.PrerequisiteInput(
137  name="bias",
138  doc="Input bias calibration.",
139  storageClass="ExposureF",
140  dimensions=["instrument", "detector"],
141  isCalibration=True,
142  )
143  dark = cT.PrerequisiteInput(
144  name='dark',
145  doc="Input dark calibration.",
146  storageClass="ExposureF",
147  dimensions=["instrument", "detector"],
148  isCalibration=True,
149  )
150  flat = cT.PrerequisiteInput(
151  name="flat",
152  doc="Input flat calibration.",
153  storageClass="ExposureF",
154  dimensions=["instrument", "physical_filter", "detector"],
155  isCalibration=True,
156  )
157  ptc = cT.PrerequisiteInput(
158  name="ptc",
159  doc="Input Photon Transfer Curve dataset",
160  storageClass="PhotonTransferCurveDataset",
161  dimensions=["instrument", "detector"],
162  isCalibration=True,
163  )
164  fringes = cT.PrerequisiteInput(
165  name="fringe",
166  doc="Input fringe calibration.",
167  storageClass="ExposureF",
168  dimensions=["instrument", "physical_filter", "detector"],
169  isCalibration=True,
170  minimum=0, # only needed for some bands, even when enabled
171  )
172  strayLightData = cT.PrerequisiteInput(
173  name='yBackground',
174  doc="Input stray light calibration.",
175  storageClass="StrayLightData",
176  dimensions=["instrument", "physical_filter", "detector"],
177  deferLoad=True,
178  isCalibration=True,
179  minimum=0, # only needed for some bands, even when enabled
180  )
181  bfKernel = cT.PrerequisiteInput(
182  name='bfKernel',
183  doc="Input brighter-fatter kernel.",
184  storageClass="NumpyArray",
185  dimensions=["instrument"],
186  isCalibration=True,
187  minimum=0, # can use either bfKernel or newBFKernel
188  )
189  newBFKernel = cT.PrerequisiteInput(
190  name='brighterFatterKernel',
191  doc="Newer complete kernel + gain solutions.",
192  storageClass="BrighterFatterKernel",
193  dimensions=["instrument", "detector"],
194  isCalibration=True,
195  minimum=0, # can use either bfKernel or newBFKernel
196  )
197  defects = cT.PrerequisiteInput(
198  name='defects',
199  doc="Input defect tables.",
200  storageClass="Defects",
201  dimensions=["instrument", "detector"],
202  isCalibration=True,
203  )
204  linearizer = cT.PrerequisiteInput(
205  name='linearizer',
206  storageClass="Linearizer",
207  doc="Linearity correction calibration.",
208  dimensions=["instrument", "detector"],
209  isCalibration=True,
210  minimum=0, # can fall back to cameraGeom
211  )
212  opticsTransmission = cT.PrerequisiteInput(
213  name="transmission_optics",
214  storageClass="TransmissionCurve",
215  doc="Transmission curve due to the optics.",
216  dimensions=["instrument"],
217  isCalibration=True,
218  )
219  filterTransmission = cT.PrerequisiteInput(
220  name="transmission_filter",
221  storageClass="TransmissionCurve",
222  doc="Transmission curve due to the filter.",
223  dimensions=["instrument", "physical_filter"],
224  isCalibration=True,
225  )
226  sensorTransmission = cT.PrerequisiteInput(
227  name="transmission_sensor",
228  storageClass="TransmissionCurve",
229  doc="Transmission curve due to the sensor.",
230  dimensions=["instrument", "detector"],
231  isCalibration=True,
232  )
233  atmosphereTransmission = cT.PrerequisiteInput(
234  name="transmission_atmosphere",
235  storageClass="TransmissionCurve",
236  doc="Transmission curve due to the atmosphere.",
237  dimensions=["instrument"],
238  isCalibration=True,
239  )
240  illumMaskedImage = cT.PrerequisiteInput(
241  name="illum",
242  doc="Input illumination correction.",
243  storageClass="MaskedImageF",
244  dimensions=["instrument", "physical_filter", "detector"],
245  isCalibration=True,
246  )
247 
248  outputExposure = cT.Output(
249  name='postISRCCD',
250  doc="Output ISR processed exposure.",
251  storageClass="Exposure",
252  dimensions=["instrument", "exposure", "detector"],
253  )
254  preInterpExposure = cT.Output(
255  name='preInterpISRCCD',
256  doc="Output ISR processed exposure, with pixels left uninterpolated.",
257  storageClass="ExposureF",
258  dimensions=["instrument", "exposure", "detector"],
259  )
260  outputOssThumbnail = cT.Output(
261  name="OssThumb",
262  doc="Output Overscan-subtracted thumbnail image.",
263  storageClass="Thumbnail",
264  dimensions=["instrument", "exposure", "detector"],
265  )
266  outputFlattenedThumbnail = cT.Output(
267  name="FlattenedThumb",
268  doc="Output flat-corrected thumbnail image.",
269  storageClass="Thumbnail",
270  dimensions=["instrument", "exposure", "detector"],
271  )
272 
273  def __init__(self, *, config=None):
274  super().__init__(config=config)
275 
276  if config.doBias is not True:
277  self.prerequisiteInputs.discard("bias")
278  if config.doLinearize is not True:
279  self.prerequisiteInputs.discard("linearizer")
280  if config.doCrosstalk is not True:
281  self.inputs.discard("crosstalkSources")
282  self.prerequisiteInputs.discard("crosstalk")
283  if config.doBrighterFatter is not True:
284  self.prerequisiteInputs.discard("bfKernel")
285  self.prerequisiteInputs.discard("newBFKernel")
286  if config.doDefect is not True:
287  self.prerequisiteInputs.discard("defects")
288  if config.doDark is not True:
289  self.prerequisiteInputs.discard("dark")
290  if config.doFlat is not True:
291  self.prerequisiteInputs.discard("flat")
292  if config.doFringe is not True:
293  self.prerequisiteInputs.discard("fringe")
294  if config.doStrayLight is not True:
295  self.prerequisiteInputs.discard("strayLightData")
296  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
297  self.prerequisiteInputs.discard("ptc")
298  if config.doAttachTransmissionCurve is not True:
299  self.prerequisiteInputs.discard("opticsTransmission")
300  self.prerequisiteInputs.discard("filterTransmission")
301  self.prerequisiteInputs.discard("sensorTransmission")
302  self.prerequisiteInputs.discard("atmosphereTransmission")
303  if config.doUseOpticsTransmission is not True:
304  self.prerequisiteInputs.discard("opticsTransmission")
305  if config.doUseFilterTransmission is not True:
306  self.prerequisiteInputs.discard("filterTransmission")
307  if config.doUseSensorTransmission is not True:
308  self.prerequisiteInputs.discard("sensorTransmission")
309  if config.doUseAtmosphereTransmission is not True:
310  self.prerequisiteInputs.discard("atmosphereTransmission")
311  if config.doIlluminationCorrection is not True:
312  self.prerequisiteInputs.discard("illumMaskedImage")
313 
314  if config.doWrite is not True:
315  self.outputs.discard("outputExposure")
316  self.outputs.discard("preInterpExposure")
317  self.outputs.discard("outputFlattenedThumbnail")
318  self.outputs.discard("outputOssThumbnail")
319  if config.doSaveInterpPixels is not True:
320  self.outputs.discard("preInterpExposure")
321  if config.qa.doThumbnailOss is not True:
322  self.outputs.discard("outputOssThumbnail")
323  if config.qa.doThumbnailFlattened is not True:
324  self.outputs.discard("outputFlattenedThumbnail")
325 
326 
327 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
328  pipelineConnections=IsrTaskConnections):
329  """Configuration parameters for IsrTask.
330 
331  Items are grouped in the order in which they are executed by the task.
332  """
333  datasetType = pexConfig.Field(
334  dtype=str,
335  doc="Dataset type for input data; users will typically leave this alone, "
336  "but camera-specific ISR tasks will override it",
337  default="raw",
338  )
339 
340  fallbackFilterName = pexConfig.Field(
341  dtype=str,
342  doc="Fallback default filter name for calibrations.",
343  optional=True
344  )
345  useFallbackDate = pexConfig.Field(
346  dtype=bool,
347  doc="Pass observation date when using fallback filter.",
348  default=False,
349  )
350  expectWcs = pexConfig.Field(
351  dtype=bool,
352  default=True,
353  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
354  )
355  fwhm = pexConfig.Field(
356  dtype=float,
357  doc="FWHM of PSF in arcseconds.",
358  default=1.0,
359  )
360  qa = pexConfig.ConfigField(
361  dtype=isrQa.IsrQaConfig,
362  doc="QA related configuration options.",
363  )
364 
365  # Image conversion configuration
366  doConvertIntToFloat = pexConfig.Field(
367  dtype=bool,
368  doc="Convert integer raw images to floating point values?",
369  default=True,
370  )
371 
372  # Saturated pixel handling.
373  doSaturation = pexConfig.Field(
374  dtype=bool,
375  doc="Mask saturated pixels? NB: this is totally independent of the"
376  " interpolation option - this is ONLY setting the bits in the mask."
377  " To have them interpolated make sure doSaturationInterpolation=True",
378  default=True,
379  )
380  saturatedMaskName = pexConfig.Field(
381  dtype=str,
382  doc="Name of mask plane to use in saturation detection and interpolation",
383  default="SAT",
384  )
385  saturation = pexConfig.Field(
386  dtype=float,
387  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
388  default=float("NaN"),
389  )
390  growSaturationFootprintSize = pexConfig.Field(
391  dtype=int,
392  doc="Number of pixels by which to grow the saturation footprints",
393  default=1,
394  )
395 
396  # Suspect pixel handling.
397  doSuspect = pexConfig.Field(
398  dtype=bool,
399  doc="Mask suspect pixels?",
400  default=False,
401  )
402  suspectMaskName = pexConfig.Field(
403  dtype=str,
404  doc="Name of mask plane to use for suspect pixels",
405  default="SUSPECT",
406  )
407  numEdgeSuspect = pexConfig.Field(
408  dtype=int,
409  doc="Number of edge pixels to be flagged as untrustworthy.",
410  default=0,
411  )
412  edgeMaskLevel = pexConfig.ChoiceField(
413  dtype=str,
414  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
415  default="DETECTOR",
416  allowed={
417  'DETECTOR': 'Mask only the edges of the full detector.',
418  'AMP': 'Mask edges of each amplifier.',
419  },
420  )
421 
422  # Initial masking options.
423  doSetBadRegions = pexConfig.Field(
424  dtype=bool,
425  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
426  default=True,
427  )
428  badStatistic = pexConfig.ChoiceField(
429  dtype=str,
430  doc="How to estimate the average value for BAD regions.",
431  default='MEANCLIP',
432  allowed={
433  "MEANCLIP": "Correct using the (clipped) mean of good data",
434  "MEDIAN": "Correct using the median of the good data",
435  },
436  )
437 
438  # Overscan subtraction configuration.
439  doOverscan = pexConfig.Field(
440  dtype=bool,
441  doc="Do overscan subtraction?",
442  default=True,
443  )
444  overscan = pexConfig.ConfigurableField(
445  target=OverscanCorrectionTask,
446  doc="Overscan subtraction task for image segments.",
447  )
448 
449  overscanFitType = pexConfig.ChoiceField(
450  dtype=str,
451  doc="The method for fitting the overscan bias level.",
452  default='MEDIAN',
453  allowed={
454  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
455  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
456  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
457  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
458  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
459  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
460  "MEAN": "Correct using the mean of the overscan region",
461  "MEANCLIP": "Correct using a clipped mean of the overscan region",
462  "MEDIAN": "Correct using the median of the overscan region",
463  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
464  },
465  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
466  " This option will no longer be used, and will be removed after v20.")
467  )
468  overscanOrder = pexConfig.Field(
469  dtype=int,
470  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
471  "or number of spline knots if overscan fit type is a spline."),
472  default=1,
473  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
474  " This option will no longer be used, and will be removed after v20.")
475  )
476  overscanNumSigmaClip = pexConfig.Field(
477  dtype=float,
478  doc="Rejection threshold (sigma) for collapsing overscan before fit",
479  default=3.0,
480  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
481  " This option will no longer be used, and will be removed after v20.")
482  )
483  overscanIsInt = pexConfig.Field(
484  dtype=bool,
485  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
486  " and overscan.FitType=MEDIAN_PER_ROW.",
487  default=True,
488  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
489  " This option will no longer be used, and will be removed after v20.")
490  )
491  # These options do not get deprecated, as they define how we slice up the image data.
492  overscanNumLeadingColumnsToSkip = pexConfig.Field(
493  dtype=int,
494  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
495  default=0,
496  )
497  overscanNumTrailingColumnsToSkip = pexConfig.Field(
498  dtype=int,
499  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
500  default=0,
501  )
502  overscanMaxDev = pexConfig.Field(
503  dtype=float,
504  doc="Maximum deviation from the median for overscan",
505  default=1000.0, check=lambda x: x > 0
506  )
507  overscanBiasJump = pexConfig.Field(
508  dtype=bool,
509  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
510  default=False,
511  )
512  overscanBiasJumpKeyword = pexConfig.Field(
513  dtype=str,
514  doc="Header keyword containing information about devices.",
515  default="NO_SUCH_KEY",
516  )
517  overscanBiasJumpDevices = pexConfig.ListField(
518  dtype=str,
519  doc="List of devices that need piecewise overscan correction.",
520  default=(),
521  )
522  overscanBiasJumpLocation = pexConfig.Field(
523  dtype=int,
524  doc="Location of bias jump along y-axis.",
525  default=0,
526  )
527 
528  # Amplifier to CCD assembly configuration
529  doAssembleCcd = pexConfig.Field(
530  dtype=bool,
531  default=True,
532  doc="Assemble amp-level exposures into a ccd-level exposure?"
533  )
534  assembleCcd = pexConfig.ConfigurableField(
535  target=AssembleCcdTask,
536  doc="CCD assembly task",
537  )
538 
539  # General calibration configuration.
540  doAssembleIsrExposures = pexConfig.Field(
541  dtype=bool,
542  default=False,
543  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
544  )
545  doTrimToMatchCalib = pexConfig.Field(
546  dtype=bool,
547  default=False,
548  doc="Trim raw data to match calibration bounding boxes?"
549  )
550 
551  # Bias subtraction.
552  doBias = pexConfig.Field(
553  dtype=bool,
554  doc="Apply bias frame correction?",
555  default=True,
556  )
557  biasDataProductName = pexConfig.Field(
558  dtype=str,
559  doc="Name of the bias data product",
560  default="bias",
561  )
562  doBiasBeforeOverscan = pexConfig.Field(
563  dtype=bool,
564  doc="Reverse order of overscan and bias correction.",
565  default=False
566  )
567 
568  # Variance construction
569  doVariance = pexConfig.Field(
570  dtype=bool,
571  doc="Calculate variance?",
572  default=True
573  )
574  gain = pexConfig.Field(
575  dtype=float,
576  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
577  default=float("NaN"),
578  )
579  readNoise = pexConfig.Field(
580  dtype=float,
581  doc="The read noise to use if no Detector is present in the Exposure",
582  default=0.0,
583  )
584  doEmpiricalReadNoise = pexConfig.Field(
585  dtype=bool,
586  default=False,
587  doc="Calculate empirical read noise instead of value from AmpInfo data?"
588  )
589  usePtcReadNoise = pexConfig.Field(
590  dtype=bool,
591  default=False,
592  doc="Use readnoise values from the Photon Transfer Curve?"
593  )
594  maskNegativeVariance = pexConfig.Field(
595  dtype=bool,
596  default=True,
597  doc="Mask pixels that claim a negative variance? This likely indicates a failure "
598  "in the measurement of the overscan at an edge due to the data falling off faster "
599  "than the overscan model can account for it."
600  )
601  negativeVarianceMaskName = pexConfig.Field(
602  dtype=str,
603  default="BAD",
604  doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
605  )
606  # Linearization.
607  doLinearize = pexConfig.Field(
608  dtype=bool,
609  doc="Correct for nonlinearity of the detector's response?",
610  default=True,
611  )
612 
613  # Crosstalk.
614  doCrosstalk = pexConfig.Field(
615  dtype=bool,
616  doc="Apply intra-CCD crosstalk correction?",
617  default=False,
618  )
619  doCrosstalkBeforeAssemble = pexConfig.Field(
620  dtype=bool,
621  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
622  default=False,
623  )
624  crosstalk = pexConfig.ConfigurableField(
625  target=CrosstalkTask,
626  doc="Intra-CCD crosstalk correction",
627  )
628 
629  # Masking options.
630  doDefect = pexConfig.Field(
631  dtype=bool,
632  doc="Apply correction for CCD defects, e.g. hot pixels?",
633  default=True,
634  )
635  doNanMasking = pexConfig.Field(
636  dtype=bool,
637  doc="Mask non-finite (NAN, inf) pixels?",
638  default=True,
639  )
640  doWidenSaturationTrails = pexConfig.Field(
641  dtype=bool,
642  doc="Widen bleed trails based on their width?",
643  default=True
644  )
645 
646  # Brighter-Fatter correction.
647  doBrighterFatter = pexConfig.Field(
648  dtype=bool,
649  default=False,
650  doc="Apply the brighter-fatter correction?"
651  )
652  brighterFatterLevel = pexConfig.ChoiceField(
653  dtype=str,
654  default="DETECTOR",
655  doc="The level at which to correct for brighter-fatter.",
656  allowed={
657  "AMP": "Every amplifier treated separately.",
658  "DETECTOR": "One kernel per detector",
659  }
660  )
661  brighterFatterMaxIter = pexConfig.Field(
662  dtype=int,
663  default=10,
664  doc="Maximum number of iterations for the brighter-fatter correction"
665  )
666  brighterFatterThreshold = pexConfig.Field(
667  dtype=float,
668  default=1000,
669  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
670  "absolute value of the difference between the current corrected image and the one "
671  "from the previous iteration summed over all the pixels."
672  )
673  brighterFatterApplyGain = pexConfig.Field(
674  dtype=bool,
675  default=True,
676  doc="Should the gain be applied when applying the brighter-fatter correction?"
677  )
678  brighterFatterMaskListToInterpolate = pexConfig.ListField(
679  dtype=str,
680  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
681  "correction.",
682  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
683  )
684  brighterFatterMaskGrowSize = pexConfig.Field(
685  dtype=int,
686  default=0,
687  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
688  "when brighter-fatter correction is applied."
689  )
690 
691  # Dark subtraction.
692  doDark = pexConfig.Field(
693  dtype=bool,
694  doc="Apply dark frame correction?",
695  default=True,
696  )
697  darkDataProductName = pexConfig.Field(
698  dtype=str,
699  doc="Name of the dark data product",
700  default="dark",
701  )
702 
703  # Camera-specific stray light removal.
704  doStrayLight = pexConfig.Field(
705  dtype=bool,
706  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
707  default=False,
708  )
709  strayLight = pexConfig.ConfigurableField(
710  target=StrayLightTask,
711  doc="y-band stray light correction"
712  )
713 
714  # Flat correction.
715  doFlat = pexConfig.Field(
716  dtype=bool,
717  doc="Apply flat field correction?",
718  default=True,
719  )
720  flatDataProductName = pexConfig.Field(
721  dtype=str,
722  doc="Name of the flat data product",
723  default="flat",
724  )
725  flatScalingType = pexConfig.ChoiceField(
726  dtype=str,
727  doc="The method for scaling the flat on the fly.",
728  default='USER',
729  allowed={
730  "USER": "Scale by flatUserScale",
731  "MEAN": "Scale by the inverse of the mean",
732  "MEDIAN": "Scale by the inverse of the median",
733  },
734  )
735  flatUserScale = pexConfig.Field(
736  dtype=float,
737  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
738  default=1.0,
739  )
740  doTweakFlat = pexConfig.Field(
741  dtype=bool,
742  doc="Tweak flats to match observed amplifier ratios?",
743  default=False
744  )
745 
746  # Amplifier normalization based on gains instead of using flats configuration.
747  doApplyGains = pexConfig.Field(
748  dtype=bool,
749  doc="Correct the amplifiers for their gains instead of applying flat correction",
750  default=False,
751  )
752  usePtcGains = pexConfig.Field(
753  dtype=bool,
754  doc="Use the gain values from the Photon Transfer Curve?",
755  default=False,
756  )
757  normalizeGains = pexConfig.Field(
758  dtype=bool,
759  doc="Normalize all the amplifiers in each CCD to have the same median value.",
760  default=False,
761  )
762 
763  # Fringe correction.
764  doFringe = pexConfig.Field(
765  dtype=bool,
766  doc="Apply fringe correction?",
767  default=True,
768  )
769  fringe = pexConfig.ConfigurableField(
770  target=FringeTask,
771  doc="Fringe subtraction task",
772  )
773  fringeAfterFlat = pexConfig.Field(
774  dtype=bool,
775  doc="Do fringe subtraction after flat-fielding?",
776  default=True,
777  )
778 
779  # Initial CCD-level background statistics options.
780  doMeasureBackground = pexConfig.Field(
781  dtype=bool,
782  doc="Measure the background level on the reduced image?",
783  default=False,
784  )
785 
786  # Camera-specific masking configuration.
787  doCameraSpecificMasking = pexConfig.Field(
788  dtype=bool,
789  doc="Mask camera-specific bad regions?",
790  default=False,
791  )
792  masking = pexConfig.ConfigurableField(
793  target=MaskingTask,
794  doc="Masking task."
795  )
796 
797  # Interpolation options.
798 
799  doInterpolate = pexConfig.Field(
800  dtype=bool,
801  doc="Interpolate masked pixels?",
802  default=True,
803  )
804  doSaturationInterpolation = pexConfig.Field(
805  dtype=bool,
806  doc="Perform interpolation over pixels masked as saturated?"
807  " NB: This is independent of doSaturation; if that is False this plane"
808  " will likely be blank, resulting in a no-op here.",
809  default=True,
810  )
811  doNanInterpolation = pexConfig.Field(
812  dtype=bool,
813  doc="Perform interpolation over pixels masked as NaN?"
814  " NB: This is independent of doNanMasking; if that is False this plane"
815  " will likely be blank, resulting in a no-op here.",
816  default=True,
817  )
818  doNanInterpAfterFlat = pexConfig.Field(
819  dtype=bool,
820  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
821  "also have to interpolate them before flat-fielding."),
822  default=False,
823  )
824  maskListToInterpolate = pexConfig.ListField(
825  dtype=str,
826  doc="List of mask planes that should be interpolated.",
827  default=['SAT', 'BAD'],
828  )
829  doSaveInterpPixels = pexConfig.Field(
830  dtype=bool,
831  doc="Save a copy of the pre-interpolated pixel values?",
832  default=False,
833  )
834 
835  # Default photometric calibration options.
836  fluxMag0T1 = pexConfig.DictField(
837  keytype=str,
838  itemtype=float,
839  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
840  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
841  ))
842  )
843  defaultFluxMag0T1 = pexConfig.Field(
844  dtype=float,
845  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
846  default=pow(10.0, 0.4*28.0)
847  )
848 
849  # Vignette correction configuration.
850  doVignette = pexConfig.Field(
851  dtype=bool,
852  doc="Apply vignetting parameters?",
853  default=False,
854  )
855  vignette = pexConfig.ConfigurableField(
856  target=VignetteTask,
857  doc="Vignetting task.",
858  )
859 
860  # Transmission curve configuration.
861  doAttachTransmissionCurve = pexConfig.Field(
862  dtype=bool,
863  default=False,
864  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
865  )
866  doUseOpticsTransmission = pexConfig.Field(
867  dtype=bool,
868  default=True,
869  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
870  )
871  doUseFilterTransmission = pexConfig.Field(
872  dtype=bool,
873  default=True,
874  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
875  )
876  doUseSensorTransmission = pexConfig.Field(
877  dtype=bool,
878  default=True,
879  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
880  )
881  doUseAtmosphereTransmission = pexConfig.Field(
882  dtype=bool,
883  default=True,
884  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
885  )
886 
887  # Illumination correction.
888  doIlluminationCorrection = pexConfig.Field(
889  dtype=bool,
890  default=False,
891  doc="Perform illumination correction?"
892  )
893  illuminationCorrectionDataProductName = pexConfig.Field(
894  dtype=str,
895  doc="Name of the illumination correction data product.",
896  default="illumcor",
897  )
898  illumScale = pexConfig.Field(
899  dtype=float,
900  doc="Scale factor for the illumination correction.",
901  default=1.0,
902  )
903  illumFilters = pexConfig.ListField(
904  dtype=str,
905  default=[],
906  doc="Only perform illumination correction for these filters."
907  )
908 
909  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
910  doWrite = pexConfig.Field(
911  dtype=bool,
912  doc="Persist postISRCCD?",
913  default=True,
914  )
915 
916  def validate(self):
917  super().validate()
918  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
919  raise ValueError("You may not specify both doFlat and doApplyGains")
920  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
921  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
922  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
923  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
924  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
925  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
926  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
927  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
928 
929 
930 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
931  """Apply common instrument signature correction algorithms to a raw frame.
932 
933  The process for correcting imaging data is very similar from
934  camera to camera. This task provides a vanilla implementation of
935  doing these corrections, including the ability to turn certain
936  corrections off if they are not needed. The inputs to the primary
937  method, `run()`, are a raw exposure to be corrected and the
938  calibration data products. The raw input is a single chip sized
939  mosaic of all amps including overscans and other non-science
940  pixels. The method `runDataRef()` identifies and defines the
941  calibration data products, and is intended for use by a
942  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
943  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
944  subclassed for different camera, although the most camera specific
945  methods have been split into subtasks that can be redirected
946  appropriately.
947 
948  The __init__ method sets up the subtasks for ISR processing, using
949  the defaults from `lsst.ip.isr`.
950 
951  Parameters
952  ----------
953  args : `list`
954  Positional arguments passed to the Task constructor. None used at this time.
955  kwargs : `dict`, optional
956  Keyword arguments passed on to the Task constructor. None used at this time.
957  """
958  ConfigClass = IsrTaskConfig
959  _DefaultName = "isr"
960 
961  def __init__(self, **kwargs):
962  super().__init__(**kwargs)
963  self.makeSubtask("assembleCcd")
964  self.makeSubtask("crosstalk")
965  self.makeSubtask("strayLight")
966  self.makeSubtask("fringe")
967  self.makeSubtask("masking")
968  self.makeSubtask("overscan")
969  self.makeSubtask("vignette")
970 
971  def runQuantum(self, butlerQC, inputRefs, outputRefs):
972  inputs = butlerQC.get(inputRefs)
973 
974  try:
975  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
976  except Exception as e:
977  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
978  (inputRefs, e))
979 
980  inputs['isGen3'] = True
981 
982  detector = inputs['ccdExposure'].getDetector()
983 
984  if self.config.doCrosstalk is True:
985  # Crosstalk sources need to be defined by the pipeline
986  # yaml if they exist.
987  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
988  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
989  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
990  else:
991  coeffVector = (self.config.crosstalk.crosstalkValues
992  if self.config.crosstalk.useConfigCoefficients else None)
993  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
994  inputs['crosstalk'] = crosstalkCalib
995  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
996  if 'crosstalkSources' not in inputs:
997  self.log.warning("No crosstalkSources found for chip with interChip terms!")
998 
999  if self.doLinearizedoLinearize(detector) is True:
1000  if 'linearizer' in inputs:
1001  if isinstance(inputs['linearizer'], dict):
1002  linearizer = linearize.Linearizer(detector=detector, log=self.log)
1003  linearizer.fromYaml(inputs['linearizer'])
1004  self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1005  elif isinstance(inputs['linearizer'], numpy.ndarray):
1006  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1007  detector=detector,
1008  log=self.log)
1009  self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1010  else:
1011  linearizer = inputs['linearizer']
1012  linearizer.log = self.log
1013  inputs['linearizer'] = linearizer
1014  else:
1015  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1016  self.log.warning("Constructing linearizer from cameraGeom information.")
1017 
1018  if self.config.doDefect is True:
1019  if "defects" in inputs and inputs['defects'] is not None:
1020  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1021  # masking expects a list of defects defined by their bounding box
1022  if not isinstance(inputs["defects"], Defects):
1023  inputs["defects"] = Defects.fromTable(inputs["defects"])
1024 
1025  # Load the correct style of brighter-fatter kernel, and repack
1026  # the information as a numpy array.
1027  if self.config.doBrighterFatter:
1028  brighterFatterKernel = inputs.pop('newBFKernel', None)
1029  if brighterFatterKernel is None:
1030  brighterFatterKernel = inputs.get('bfKernel', None)
1031 
1032  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1033  # This is a ISR calib kernel
1034  detName = detector.getName()
1035  level = brighterFatterKernel.level
1036 
1037  # This is expected to be a dictionary of amp-wise gains.
1038  inputs['bfGains'] = brighterFatterKernel.gain
1039  if self.config.brighterFatterLevel == 'DETECTOR':
1040  if level == 'DETECTOR':
1041  if detName in brighterFatterKernel.detKernels:
1042  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1043  else:
1044  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1045  elif level == 'AMP':
1046  self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1047  "fatter kernels.")
1048  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1049  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1050  elif self.config.brighterFatterLevel == 'AMP':
1051  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1052 
1053  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1054  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1055  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1056  expId=expId,
1057  assembler=self.assembleCcd
1058  if self.config.doAssembleIsrExposures else None)
1059  else:
1060  inputs['fringes'] = pipeBase.Struct(fringes=None)
1061 
1062  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1063  if 'strayLightData' not in inputs:
1064  inputs['strayLightData'] = None
1065 
1066  outputs = self.runrun(**inputs)
1067  butlerQC.put(outputs, outputRefs)
1068 
1069  def readIsrData(self, dataRef, rawExposure):
1070  """Retrieve necessary frames for instrument signature removal.
1071 
1072  Pre-fetching all required ISR data products limits the IO
1073  required by the ISR. Any conflict between the calibration data
1074  available and that needed for ISR is also detected prior to
1075  doing processing, allowing it to fail quickly.
1076 
1077  Parameters
1078  ----------
1079  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1080  Butler reference of the detector data to be processed
1081  rawExposure : `afw.image.Exposure`
1082  The raw exposure that will later be corrected with the
1083  retrieved calibration data; should not be modified in this
1084  method.
1085 
1086  Returns
1087  -------
1088  result : `lsst.pipe.base.Struct`
1089  Result struct with components (which may be `None`):
1090  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1091  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1092  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1093  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1094  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1095  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1096  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1097  - ``fringes``: `lsst.pipe.base.Struct` with components:
1098  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1099  - ``seed``: random seed derived from the ccdExposureId for random
1100  number generator (`uint32`).
1101  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1102  A ``TransmissionCurve`` that represents the throughput of the optics,
1103  to be evaluated in focal-plane coordinates.
1104  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1105  A ``TransmissionCurve`` that represents the throughput of the filter
1106  itself, to be evaluated in focal-plane coordinates.
1107  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1108  A ``TransmissionCurve`` that represents the throughput of the sensor
1109  itself, to be evaluated in post-assembly trimmed detector coordinates.
1110  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1111  A ``TransmissionCurve`` that represents the throughput of the
1112  atmosphere, assumed to be spatially constant.
1113  - ``strayLightData`` : `object`
1114  An opaque object containing calibration information for
1115  stray-light correction. If `None`, no correction will be
1116  performed.
1117  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1118 
1119  Raises
1120  ------
1121  NotImplementedError :
1122  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1123  """
1124  try:
1125  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1126  dateObs = dateObs.toPython().isoformat()
1127  except RuntimeError:
1128  self.log.warning("Unable to identify dateObs for rawExposure.")
1129  dateObs = None
1130 
1131  ccd = rawExposure.getDetector()
1132  filterLabel = rawExposure.getFilterLabel()
1133  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1134  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1135  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1136  if self.config.doBias else None)
1137  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1138  linearizer = (dataRef.get("linearizer", immediate=True)
1139  if self.doLinearizedoLinearize(ccd) else None)
1140  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1141  linearizer.log = self.log
1142  if isinstance(linearizer, numpy.ndarray):
1143  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1144 
1145  crosstalkCalib = None
1146  if self.config.doCrosstalk:
1147  try:
1148  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1149  except NoResults:
1150  coeffVector = (self.config.crosstalk.crosstalkValues
1151  if self.config.crosstalk.useConfigCoefficients else None)
1152  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1153  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1154  if self.config.doCrosstalk else None)
1155 
1156  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1157  if self.config.doDark else None)
1158  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1159  dateObs=dateObs)
1160  if self.config.doFlat else None)
1161 
1162  brighterFatterKernel = None
1163  brighterFatterGains = None
1164  if self.config.doBrighterFatter is True:
1165  try:
1166  # Use the new-style cp_pipe version of the kernel if it exists
1167  # If using a new-style kernel, always use the self-consistent
1168  # gains, i.e. the ones inside the kernel object itself
1169  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1170  brighterFatterGains = brighterFatterKernel.gain
1171  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1172  except NoResults:
1173  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1174  brighterFatterKernel = dataRef.get("bfKernel")
1175  self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1176  except NoResults:
1177  brighterFatterKernel = None
1178  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1179  # If the kernel is not an ndarray, it's the cp_pipe version
1180  # so extract the kernel for this detector, or raise an error
1181  if self.config.brighterFatterLevel == 'DETECTOR':
1182  if brighterFatterKernel.detKernels:
1183  brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1184  else:
1185  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1186  else:
1187  # TODO DM-15631 for implementing this
1188  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1189 
1190  defectList = (dataRef.get("defects")
1191  if self.config.doDefect else None)
1192  expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1193  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1194  if self.config.doAssembleIsrExposures else None)
1195  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1196  else pipeBase.Struct(fringes=None))
1197 
1198  if self.config.doAttachTransmissionCurve:
1199  opticsTransmission = (dataRef.get("transmission_optics")
1200  if self.config.doUseOpticsTransmission else None)
1201  filterTransmission = (dataRef.get("transmission_filter")
1202  if self.config.doUseFilterTransmission else None)
1203  sensorTransmission = (dataRef.get("transmission_sensor")
1204  if self.config.doUseSensorTransmission else None)
1205  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1206  if self.config.doUseAtmosphereTransmission else None)
1207  else:
1208  opticsTransmission = None
1209  filterTransmission = None
1210  sensorTransmission = None
1211  atmosphereTransmission = None
1212 
1213  if self.config.doStrayLight:
1214  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1215  else:
1216  strayLightData = None
1217 
1218  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1219  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1220  if (self.config.doIlluminationCorrection
1221  and physicalFilter in self.config.illumFilters)
1222  else None)
1223 
1224  # Struct should include only kwargs to run()
1225  return pipeBase.Struct(bias=biasExposure,
1226  linearizer=linearizer,
1227  crosstalk=crosstalkCalib,
1228  crosstalkSources=crosstalkSources,
1229  dark=darkExposure,
1230  flat=flatExposure,
1231  bfKernel=brighterFatterKernel,
1232  bfGains=brighterFatterGains,
1233  defects=defectList,
1234  fringes=fringeStruct,
1235  opticsTransmission=opticsTransmission,
1236  filterTransmission=filterTransmission,
1237  sensorTransmission=sensorTransmission,
1238  atmosphereTransmission=atmosphereTransmission,
1239  strayLightData=strayLightData,
1240  illumMaskedImage=illumMaskedImage
1241  )
1242 
1243  @pipeBase.timeMethod
1244  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1245  crosstalk=None, crosstalkSources=None,
1246  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1247  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1248  sensorTransmission=None, atmosphereTransmission=None,
1249  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1250  isGen3=False,
1251  ):
1252  """Perform instrument signature removal on an exposure.
1253 
1254  Steps included in the ISR processing, in order performed, are:
1255  - saturation and suspect pixel masking
1256  - overscan subtraction
1257  - CCD assembly of individual amplifiers
1258  - bias subtraction
1259  - variance image construction
1260  - linearization of non-linear response
1261  - crosstalk masking
1262  - brighter-fatter correction
1263  - dark subtraction
1264  - fringe correction
1265  - stray light subtraction
1266  - flat correction
1267  - masking of known defects and camera specific features
1268  - vignette calculation
1269  - appending transmission curve and distortion model
1270 
1271  Parameters
1272  ----------
1273  ccdExposure : `lsst.afw.image.Exposure`
1274  The raw exposure that is to be run through ISR. The
1275  exposure is modified by this method.
1276  camera : `lsst.afw.cameraGeom.Camera`, optional
1277  The camera geometry for this exposure. Required if ``isGen3`` is
1278  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1279  ``flat`` does not have an associated detector.
1280  bias : `lsst.afw.image.Exposure`, optional
1281  Bias calibration frame.
1282  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1283  Functor for linearization.
1284  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1285  Calibration for crosstalk.
1286  crosstalkSources : `list`, optional
1287  List of possible crosstalk sources.
1288  dark : `lsst.afw.image.Exposure`, optional
1289  Dark calibration frame.
1290  flat : `lsst.afw.image.Exposure`, optional
1291  Flat calibration frame.
1292  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1293  Photon transfer curve dataset, with, e.g., gains
1294  and read noise.
1295  bfKernel : `numpy.ndarray`, optional
1296  Brighter-fatter kernel.
1297  bfGains : `dict` of `float`, optional
1298  Gains used to override the detector's nominal gains for the
1299  brighter-fatter correction. A dict keyed by amplifier name for
1300  the detector in question.
1301  defects : `lsst.ip.isr.Defects`, optional
1302  List of defects.
1303  fringes : `lsst.pipe.base.Struct`, optional
1304  Struct containing the fringe correction data, with
1305  elements:
1306  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1307  - ``seed``: random seed derived from the ccdExposureId for random
1308  number generator (`uint32`)
1309  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1310  A ``TransmissionCurve`` that represents the throughput of the optics,
1311  to be evaluated in focal-plane coordinates.
1312  filterTransmission : `lsst.afw.image.TransmissionCurve`
1313  A ``TransmissionCurve`` that represents the throughput of the filter
1314  itself, to be evaluated in focal-plane coordinates.
1315  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1316  A ``TransmissionCurve`` that represents the throughput of the sensor
1317  itself, to be evaluated in post-assembly trimmed detector coordinates.
1318  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1319  A ``TransmissionCurve`` that represents the throughput of the
1320  atmosphere, assumed to be spatially constant.
1321  detectorNum : `int`, optional
1322  The integer number for the detector to process.
1323  isGen3 : bool, optional
1324  Flag this call to run() as using the Gen3 butler environment.
1325  strayLightData : `object`, optional
1326  Opaque object containing calibration information for stray-light
1327  correction. If `None`, no correction will be performed.
1328  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1329  Illumination correction image.
1330 
1331  Returns
1332  -------
1333  result : `lsst.pipe.base.Struct`
1334  Result struct with component:
1335  - ``exposure`` : `afw.image.Exposure`
1336  The fully ISR corrected exposure.
1337  - ``outputExposure`` : `afw.image.Exposure`
1338  An alias for `exposure`
1339  - ``ossThumb`` : `numpy.ndarray`
1340  Thumbnail image of the exposure after overscan subtraction.
1341  - ``flattenedThumb`` : `numpy.ndarray`
1342  Thumbnail image of the exposure after flat-field correction.
1343 
1344  Raises
1345  ------
1346  RuntimeError
1347  Raised if a configuration option is set to True, but the
1348  required calibration data has not been specified.
1349 
1350  Notes
1351  -----
1352  The current processed exposure can be viewed by setting the
1353  appropriate lsstDebug entries in the `debug.display`
1354  dictionary. The names of these entries correspond to some of
1355  the IsrTaskConfig Boolean options, with the value denoting the
1356  frame to use. The exposure is shown inside the matching
1357  option check and after the processing of that step has
1358  finished. The steps with debug points are:
1359 
1360  doAssembleCcd
1361  doBias
1362  doCrosstalk
1363  doBrighterFatter
1364  doDark
1365  doFringe
1366  doStrayLight
1367  doFlat
1368 
1369  In addition, setting the "postISRCCD" entry displays the
1370  exposure after all ISR processing has finished.
1371 
1372  """
1373 
1374  if isGen3 is True:
1375  # Gen3 currently cannot automatically do configuration overrides.
1376  # DM-15257 looks to discuss this issue.
1377  # Configure input exposures;
1378  if detectorNum is None:
1379  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1380 
1381  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1382  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1383  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1384  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1385  else:
1386  if isinstance(ccdExposure, ButlerDataRef):
1387  return self.runDataRefrunDataRef(ccdExposure)
1388 
1389  ccd = ccdExposure.getDetector()
1390  filterLabel = ccdExposure.getFilterLabel()
1391  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1392 
1393  if not ccd:
1394  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1395  ccd = [FakeAmp(ccdExposure, self.config)]
1396 
1397  # Validate Input
1398  if self.config.doBias and bias is None:
1399  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1400  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1401  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1402  if self.config.doBrighterFatter and bfKernel is None:
1403  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1404  if self.config.doDark and dark is None:
1405  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1406  if self.config.doFlat and flat is None:
1407  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1408  if self.config.doDefect and defects is None:
1409  raise RuntimeError("Must supply defects if config.doDefect=True.")
1410  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1411  and fringes.fringes is None):
1412  # The `fringes` object needs to be a pipeBase.Struct, as
1413  # we use it as a `dict` for the parameters of
1414  # `FringeTask.run()`. The `fringes.fringes` `list` may
1415  # not be `None` if `doFringe=True`. Otherwise, raise.
1416  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1417  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1418  and illumMaskedImage is None):
1419  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1420 
1421  # Begin ISR processing.
1422  if self.config.doConvertIntToFloat:
1423  self.log.info("Converting exposure to floating point values.")
1424  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1425 
1426  if self.config.doBias and self.config.doBiasBeforeOverscan:
1427  self.log.info("Applying bias correction.")
1428  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1429  trimToFit=self.config.doTrimToMatchCalib)
1430  self.debugViewdebugView(ccdExposure, "doBias")
1431 
1432  # Amplifier level processing.
1433  overscans = []
1434  for amp in ccd:
1435  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1436  if ccdExposure.getBBox().contains(amp.getBBox()):
1437  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1438  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1439 
1440  if self.config.doOverscan and not badAmp:
1441  # Overscan correction on amp-by-amp basis.
1442  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1443  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1444  if overscanResults is not None and \
1445  self.config.qa is not None and self.config.qa.saveStats is True:
1446  if isinstance(overscanResults.overscanFit, float):
1447  qaMedian = overscanResults.overscanFit
1448  qaStdev = float("NaN")
1449  else:
1450  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1451  afwMath.MEDIAN | afwMath.STDEVCLIP)
1452  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1453  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1454 
1455  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1456  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1457  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1458  amp.getName(), qaMedian, qaStdev)
1459 
1460  # Residuals after overscan correction
1461  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1462  afwMath.MEDIAN | afwMath.STDEVCLIP)
1463  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1464  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1465 
1466  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1467  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1468  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1469  amp.getName(), qaMedianAfter, qaStdevAfter)
1470 
1471  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1472  else:
1473  if badAmp:
1474  self.log.warning("Amplifier %s is bad.", amp.getName())
1475  overscanResults = None
1476 
1477  overscans.append(overscanResults if overscanResults is not None else None)
1478  else:
1479  self.log.info("Skipped OSCAN for %s.", amp.getName())
1480 
1481  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1482  self.log.info("Applying crosstalk correction.")
1483  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1484  crosstalkSources=crosstalkSources, camera=camera)
1485  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1486 
1487  if self.config.doAssembleCcd:
1488  self.log.info("Assembling CCD from amplifiers.")
1489  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1490 
1491  if self.config.expectWcs and not ccdExposure.getWcs():
1492  self.log.warning("No WCS found in input exposure.")
1493  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1494 
1495  ossThumb = None
1496  if self.config.qa.doThumbnailOss:
1497  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1498 
1499  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1500  self.log.info("Applying bias correction.")
1501  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1502  trimToFit=self.config.doTrimToMatchCalib)
1503  self.debugViewdebugView(ccdExposure, "doBias")
1504 
1505  if self.config.doVariance:
1506  for amp, overscanResults in zip(ccd, overscans):
1507  if ccdExposure.getBBox().contains(amp.getBBox()):
1508  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1509  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1510  if overscanResults is not None:
1511  self.updateVarianceupdateVariance(ampExposure, amp,
1512  overscanImage=overscanResults.overscanImage,
1513  ptcDataset=ptc)
1514  else:
1515  self.updateVarianceupdateVariance(ampExposure, amp,
1516  overscanImage=None,
1517  ptcDataset=ptc)
1518  if self.config.qa is not None and self.config.qa.saveStats is True:
1519  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1520  afwMath.MEDIAN | afwMath.STDEVCLIP)
1521  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1522  qaStats.getValue(afwMath.MEDIAN))
1523  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1524  qaStats.getValue(afwMath.STDEVCLIP))
1525  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1526  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1527  qaStats.getValue(afwMath.STDEVCLIP))
1528  if self.config.maskNegativeVariance:
1529  self.maskNegativeVariancemaskNegativeVariance(ccdExposure)
1530 
1531  if self.doLinearizedoLinearize(ccd):
1532  self.log.info("Applying linearizer.")
1533  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1534  detector=ccd, log=self.log)
1535 
1536  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1537  self.log.info("Applying crosstalk correction.")
1538  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1539  crosstalkSources=crosstalkSources, isTrimmed=True)
1540  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1541 
1542  # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1543  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1544  if self.config.doDefect:
1545  self.log.info("Masking defects.")
1546  self.maskDefectmaskDefect(ccdExposure, defects)
1547 
1548  if self.config.numEdgeSuspect > 0:
1549  self.log.info("Masking edges as SUSPECT.")
1550  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1551  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1552 
1553  if self.config.doNanMasking:
1554  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1555  self.maskNanmaskNan(ccdExposure)
1556 
1557  if self.config.doWidenSaturationTrails:
1558  self.log.info("Widening saturation trails.")
1559  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1560 
1561  if self.config.doCameraSpecificMasking:
1562  self.log.info("Masking regions for camera specific reasons.")
1563  self.masking.run(ccdExposure)
1564 
1565  if self.config.doBrighterFatter:
1566  # We need to apply flats and darks before we can interpolate, and we
1567  # need to interpolate before we do B-F, but we do B-F without the
1568  # flats and darks applied so we can work in units of electrons or holes.
1569  # This context manager applies and then removes the darks and flats.
1570  #
1571  # We also do not want to interpolate values here, so operate on temporary
1572  # images so we can apply only the BF-correction and roll back the
1573  # interpolation.
1574  interpExp = ccdExposure.clone()
1575  with self.flatContextflatContext(interpExp, flat, dark):
1576  isrFunctions.interpolateFromMask(
1577  maskedImage=interpExp.getMaskedImage(),
1578  fwhm=self.config.fwhm,
1579  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1580  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1581  )
1582  bfExp = interpExp.clone()
1583 
1584  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1585  type(bfKernel), type(bfGains))
1586  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1587  self.config.brighterFatterMaxIter,
1588  self.config.brighterFatterThreshold,
1589  self.config.brighterFatterApplyGain,
1590  bfGains)
1591  if bfResults[1] == self.config.brighterFatterMaxIter:
1592  self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1593  bfResults[0])
1594  else:
1595  self.log.info("Finished brighter-fatter correction in %d iterations.",
1596  bfResults[1])
1597  image = ccdExposure.getMaskedImage().getImage()
1598  bfCorr = bfExp.getMaskedImage().getImage()
1599  bfCorr -= interpExp.getMaskedImage().getImage()
1600  image += bfCorr
1601 
1602  # Applying the brighter-fatter correction applies a
1603  # convolution to the science image. At the edges this
1604  # convolution may not have sufficient valid pixels to
1605  # produce a valid correction. Mark pixels within the size
1606  # of the brighter-fatter kernel as EDGE to warn of this
1607  # fact.
1608  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1609  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1610  maskPlane="EDGE")
1611 
1612  if self.config.brighterFatterMaskGrowSize > 0:
1613  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1614  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1615  isrFunctions.growMasks(ccdExposure.getMask(),
1616  radius=self.config.brighterFatterMaskGrowSize,
1617  maskNameList=maskPlane,
1618  maskValue=maskPlane)
1619 
1620  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1621 
1622  if self.config.doDark:
1623  self.log.info("Applying dark correction.")
1624  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1625  self.debugViewdebugView(ccdExposure, "doDark")
1626 
1627  if self.config.doFringe and not self.config.fringeAfterFlat:
1628  self.log.info("Applying fringe correction before flat.")
1629  self.fringe.run(ccdExposure, **fringes.getDict())
1630  self.debugViewdebugView(ccdExposure, "doFringe")
1631 
1632  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1633  self.log.info("Checking strayLight correction.")
1634  self.strayLight.run(ccdExposure, strayLightData)
1635  self.debugViewdebugView(ccdExposure, "doStrayLight")
1636 
1637  if self.config.doFlat:
1638  self.log.info("Applying flat correction.")
1639  self.flatCorrectionflatCorrection(ccdExposure, flat)
1640  self.debugViewdebugView(ccdExposure, "doFlat")
1641 
1642  if self.config.doApplyGains:
1643  self.log.info("Applying gain correction instead of flat.")
1644  if self.config.usePtcGains:
1645  self.log.info("Using gains from the Photon Transfer Curve.")
1646  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1647  ptcGains=ptc.gain)
1648  else:
1649  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1650 
1651  if self.config.doFringe and self.config.fringeAfterFlat:
1652  self.log.info("Applying fringe correction after flat.")
1653  self.fringe.run(ccdExposure, **fringes.getDict())
1654 
1655  if self.config.doVignette:
1656  self.log.info("Constructing Vignette polygon.")
1657  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1658 
1659  if self.config.vignette.doWriteVignettePolygon:
1660  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1661 
1662  if self.config.doAttachTransmissionCurve:
1663  self.log.info("Adding transmission curves.")
1664  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1665  filterTransmission=filterTransmission,
1666  sensorTransmission=sensorTransmission,
1667  atmosphereTransmission=atmosphereTransmission)
1668 
1669  flattenedThumb = None
1670  if self.config.qa.doThumbnailFlattened:
1671  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1672 
1673  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1674  self.log.info("Performing illumination correction.")
1675  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1676  illumMaskedImage, illumScale=self.config.illumScale,
1677  trimToFit=self.config.doTrimToMatchCalib)
1678 
1679  preInterpExp = None
1680  if self.config.doSaveInterpPixels:
1681  preInterpExp = ccdExposure.clone()
1682 
1683  # Reset and interpolate bad pixels.
1684  #
1685  # Large contiguous bad regions (which should have the BAD mask
1686  # bit set) should have their values set to the image median.
1687  # This group should include defects and bad amplifiers. As the
1688  # area covered by these defects are large, there's little
1689  # reason to expect that interpolation would provide a more
1690  # useful value.
1691  #
1692  # Smaller defects can be safely interpolated after the larger
1693  # regions have had their pixel values reset. This ensures
1694  # that the remaining defects adjacent to bad amplifiers (as an
1695  # example) do not attempt to interpolate extreme values.
1696  if self.config.doSetBadRegions:
1697  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1698  if badPixelCount > 0:
1699  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1700 
1701  if self.config.doInterpolate:
1702  self.log.info("Interpolating masked pixels.")
1703  isrFunctions.interpolateFromMask(
1704  maskedImage=ccdExposure.getMaskedImage(),
1705  fwhm=self.config.fwhm,
1706  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1707  maskNameList=list(self.config.maskListToInterpolate)
1708  )
1709 
1710  self.roughZeroPointroughZeroPoint(ccdExposure)
1711 
1712  if self.config.doMeasureBackground:
1713  self.log.info("Measuring background level.")
1714  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1715 
1716  if self.config.qa is not None and self.config.qa.saveStats is True:
1717  for amp in ccd:
1718  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1719  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1720  afwMath.MEDIAN | afwMath.STDEVCLIP)
1721  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1722  qaStats.getValue(afwMath.MEDIAN))
1723  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1724  qaStats.getValue(afwMath.STDEVCLIP))
1725  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1726  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1727  qaStats.getValue(afwMath.STDEVCLIP))
1728 
1729  self.debugViewdebugView(ccdExposure, "postISRCCD")
1730 
1731  return pipeBase.Struct(
1732  exposure=ccdExposure,
1733  ossThumb=ossThumb,
1734  flattenedThumb=flattenedThumb,
1735 
1736  preInterpExposure=preInterpExp,
1737  outputExposure=ccdExposure,
1738  outputOssThumbnail=ossThumb,
1739  outputFlattenedThumbnail=flattenedThumb,
1740  )
1741 
1742  @pipeBase.timeMethod
1743  def runDataRef(self, sensorRef):
1744  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1745 
1746  This method contains the `CmdLineTask` interface to the ISR
1747  processing. All IO is handled here, freeing the `run()` method
1748  to manage only pixel-level calculations. The steps performed
1749  are:
1750  - Read in necessary detrending/isr/calibration data.
1751  - Process raw exposure in `run()`.
1752  - Persist the ISR-corrected exposure as "postISRCCD" if
1753  config.doWrite=True.
1754 
1755  Parameters
1756  ----------
1757  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1758  DataRef of the detector data to be processed
1759 
1760  Returns
1761  -------
1762  result : `lsst.pipe.base.Struct`
1763  Result struct with component:
1764  - ``exposure`` : `afw.image.Exposure`
1765  The fully ISR corrected exposure.
1766 
1767  Raises
1768  ------
1769  RuntimeError
1770  Raised if a configuration option is set to True, but the
1771  required calibration data does not exist.
1772 
1773  """
1774  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1775 
1776  ccdExposure = sensorRef.get(self.config.datasetType)
1777 
1778  camera = sensorRef.get("camera")
1779  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1780 
1781  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1782 
1783  if self.config.doWrite:
1784  sensorRef.put(result.exposure, "postISRCCD")
1785  if result.preInterpExposure is not None:
1786  sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1787  if result.ossThumb is not None:
1788  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1789  if result.flattenedThumb is not None:
1790  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1791 
1792  return result
1793 
1794  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1795  """Retrieve a calibration dataset for removing instrument signature.
1796 
1797  Parameters
1798  ----------
1799 
1800  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1801  DataRef of the detector data to find calibration datasets
1802  for.
1803  datasetType : `str`
1804  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1805  dateObs : `str`, optional
1806  Date of the observation. Used to correct butler failures
1807  when using fallback filters.
1808  immediate : `Bool`
1809  If True, disable butler proxies to enable error handling
1810  within this routine.
1811 
1812  Returns
1813  -------
1814  exposure : `lsst.afw.image.Exposure`
1815  Requested calibration frame.
1816 
1817  Raises
1818  ------
1819  RuntimeError
1820  Raised if no matching calibration frame can be found.
1821  """
1822  try:
1823  exp = dataRef.get(datasetType, immediate=immediate)
1824  except Exception as exc1:
1825  if not self.config.fallbackFilterName:
1826  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1827  try:
1828  if self.config.useFallbackDate and dateObs:
1829  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1830  dateObs=dateObs, immediate=immediate)
1831  else:
1832  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1833  except Exception as exc2:
1834  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1835  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1836  self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1837 
1838  if self.config.doAssembleIsrExposures:
1839  exp = self.assembleCcd.assembleCcd(exp)
1840  return exp
1841 
1842  def ensureExposure(self, inputExp, camera, detectorNum):
1843  """Ensure that the data returned by Butler is a fully constructed exposure.
1844 
1845  ISR requires exposure-level image data for historical reasons, so if we did
1846  not recieve that from Butler, construct it from what we have, modifying the
1847  input in place.
1848 
1849  Parameters
1850  ----------
1851  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1852  `lsst.afw.image.ImageF`
1853  The input data structure obtained from Butler.
1854  camera : `lsst.afw.cameraGeom.camera`
1855  The camera associated with the image. Used to find the appropriate
1856  detector.
1857  detectorNum : `int`
1858  The detector this exposure should match.
1859 
1860  Returns
1861  -------
1862  inputExp : `lsst.afw.image.Exposure`
1863  The re-constructed exposure, with appropriate detector parameters.
1864 
1865  Raises
1866  ------
1867  TypeError
1868  Raised if the input data cannot be used to construct an exposure.
1869  """
1870  if isinstance(inputExp, afwImage.DecoratedImageU):
1871  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1872  elif isinstance(inputExp, afwImage.ImageF):
1873  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1874  elif isinstance(inputExp, afwImage.MaskedImageF):
1875  inputExp = afwImage.makeExposure(inputExp)
1876  elif isinstance(inputExp, afwImage.Exposure):
1877  pass
1878  elif inputExp is None:
1879  # Assume this will be caught by the setup if it is a problem.
1880  return inputExp
1881  else:
1882  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1883  (type(inputExp), ))
1884 
1885  if inputExp.getDetector() is None:
1886  inputExp.setDetector(camera[detectorNum])
1887 
1888  return inputExp
1889 
1890  def convertIntToFloat(self, exposure):
1891  """Convert exposure image from uint16 to float.
1892 
1893  If the exposure does not need to be converted, the input is
1894  immediately returned. For exposures that are converted to use
1895  floating point pixels, the variance is set to unity and the
1896  mask to zero.
1897 
1898  Parameters
1899  ----------
1900  exposure : `lsst.afw.image.Exposure`
1901  The raw exposure to be converted.
1902 
1903  Returns
1904  -------
1905  newexposure : `lsst.afw.image.Exposure`
1906  The input ``exposure``, converted to floating point pixels.
1907 
1908  Raises
1909  ------
1910  RuntimeError
1911  Raised if the exposure type cannot be converted to float.
1912 
1913  """
1914  if isinstance(exposure, afwImage.ExposureF):
1915  # Nothing to be done
1916  self.log.debug("Exposure already of type float.")
1917  return exposure
1918  if not hasattr(exposure, "convertF"):
1919  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1920 
1921  newexposure = exposure.convertF()
1922  newexposure.variance[:] = 1
1923  newexposure.mask[:] = 0x0
1924 
1925  return newexposure
1926 
1927  def maskAmplifier(self, ccdExposure, amp, defects):
1928  """Identify bad amplifiers, saturated and suspect pixels.
1929 
1930  Parameters
1931  ----------
1932  ccdExposure : `lsst.afw.image.Exposure`
1933  Input exposure to be masked.
1934  amp : `lsst.afw.table.AmpInfoCatalog`
1935  Catalog of parameters defining the amplifier on this
1936  exposure to mask.
1937  defects : `lsst.ip.isr.Defects`
1938  List of defects. Used to determine if the entire
1939  amplifier is bad.
1940 
1941  Returns
1942  -------
1943  badAmp : `Bool`
1944  If this is true, the entire amplifier area is covered by
1945  defects and unusable.
1946 
1947  """
1948  maskedImage = ccdExposure.getMaskedImage()
1949 
1950  badAmp = False
1951 
1952  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1953  # comparison with current defects definition.
1954  if defects is not None:
1955  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1956 
1957  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1958  # association with pixels in current ccdExposure).
1959  if badAmp:
1960  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1961  afwImage.PARENT)
1962  maskView = dataView.getMask()
1963  maskView |= maskView.getPlaneBitMask("BAD")
1964  del maskView
1965  return badAmp
1966 
1967  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1968  # Saturation and suspect pixels can be masked now, though.
1969  limits = dict()
1970  if self.config.doSaturation and not badAmp:
1971  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1972  if self.config.doSuspect and not badAmp:
1973  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1974  if math.isfinite(self.config.saturation):
1975  limits.update({self.config.saturatedMaskName: self.config.saturation})
1976 
1977  for maskName, maskThreshold in limits.items():
1978  if not math.isnan(maskThreshold):
1979  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1980  isrFunctions.makeThresholdMask(
1981  maskedImage=dataView,
1982  threshold=maskThreshold,
1983  growFootprints=0,
1984  maskName=maskName
1985  )
1986 
1987  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1988  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1989  afwImage.PARENT)
1990  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1991  self.config.suspectMaskName])
1992  if numpy.all(maskView.getArray() & maskVal > 0):
1993  badAmp = True
1994  maskView |= maskView.getPlaneBitMask("BAD")
1995 
1996  return badAmp
1997 
1998  def overscanCorrection(self, ccdExposure, amp):
1999  """Apply overscan correction in place.
2000 
2001  This method does initial pixel rejection of the overscan
2002  region. The overscan can also be optionally segmented to
2003  allow for discontinuous overscan responses to be fit
2004  separately. The actual overscan subtraction is performed by
2005  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2006  which is called here after the amplifier is preprocessed.
2007 
2008  Parameters
2009  ----------
2010  ccdExposure : `lsst.afw.image.Exposure`
2011  Exposure to have overscan correction performed.
2012  amp : `lsst.afw.cameraGeom.Amplifer`
2013  The amplifier to consider while correcting the overscan.
2014 
2015  Returns
2016  -------
2017  overscanResults : `lsst.pipe.base.Struct`
2018  Result struct with components:
2019  - ``imageFit`` : scalar or `lsst.afw.image.Image`
2020  Value or fit subtracted from the amplifier image data.
2021  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2022  Value or fit subtracted from the overscan image data.
2023  - ``overscanImage`` : `lsst.afw.image.Image`
2024  Image of the overscan region with the overscan
2025  correction applied. This quantity is used to estimate
2026  the amplifier read noise empirically.
2027 
2028  Raises
2029  ------
2030  RuntimeError
2031  Raised if the ``amp`` does not contain raw pixel information.
2032 
2033  See Also
2034  --------
2035  lsst.ip.isr.isrFunctions.overscanCorrection
2036  """
2037  if amp.getRawHorizontalOverscanBBox().isEmpty():
2038  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2039  return None
2040 
2041  statControl = afwMath.StatisticsControl()
2042  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2043 
2044  # Determine the bounding boxes
2045  dataBBox = amp.getRawDataBBox()
2046  oscanBBox = amp.getRawHorizontalOverscanBBox()
2047  dx0 = 0
2048  dx1 = 0
2049 
2050  prescanBBox = amp.getRawPrescanBBox()
2051  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2052  dx0 += self.config.overscanNumLeadingColumnsToSkip
2053  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2054  else:
2055  dx0 += self.config.overscanNumTrailingColumnsToSkip
2056  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2057 
2058  # Determine if we need to work on subregions of the amplifier and overscan.
2059  imageBBoxes = []
2060  overscanBBoxes = []
2061 
2062  if ((self.config.overscanBiasJump
2063  and self.config.overscanBiasJumpLocation)
2064  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2065  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2066  self.config.overscanBiasJumpDevices)):
2067  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2068  yLower = self.config.overscanBiasJumpLocation
2069  yUpper = dataBBox.getHeight() - yLower
2070  else:
2071  yUpper = self.config.overscanBiasJumpLocation
2072  yLower = dataBBox.getHeight() - yUpper
2073 
2074  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2075  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2076  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2077  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2078  yLower)))
2079 
2080  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2081  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2082  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2083  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2084  yUpper)))
2085  else:
2086  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2087  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2088  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2089  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2090  oscanBBox.getHeight())))
2091 
2092  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2093  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2094  ampImage = ccdExposure.maskedImage[imageBBox]
2095  overscanImage = ccdExposure.maskedImage[overscanBBox]
2096 
2097  overscanArray = overscanImage.image.array
2098  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2099  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2100  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2101 
2102  statControl = afwMath.StatisticsControl()
2103  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2104 
2105  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2106 
2107  # Measure average overscan levels and record them in the metadata.
2108  levelStat = afwMath.MEDIAN
2109  sigmaStat = afwMath.STDEVCLIP
2110 
2111  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2112  self.config.qa.flatness.nIter)
2113  metadata = ccdExposure.getMetadata()
2114  ampNum = amp.getName()
2115  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2116  if isinstance(overscanResults.overscanFit, float):
2117  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2118  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2119  else:
2120  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2121  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2122  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2123 
2124  return overscanResults
2125 
2126  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2127  """Set the variance plane using the gain and read noise
2128 
2129  The read noise is calculated from the ``overscanImage`` if the
2130  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2131  the value from the amplifier data is used.
2132 
2133  Parameters
2134  ----------
2135  ampExposure : `lsst.afw.image.Exposure`
2136  Exposure to process.
2137  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2138  Amplifier detector data.
2139  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2140  Image of overscan, required only for empirical read noise.
2141  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2142  PTC dataset containing the gains and read noise.
2143 
2144 
2145  Raises
2146  ------
2147  RuntimeError
2148  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2149  are ``True``, but ptcDataset is not provided.
2150 
2151  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2152  ``overscanImage`` is ``None``.
2153 
2154  See also
2155  --------
2156  lsst.ip.isr.isrFunctions.updateVariance
2157  """
2158  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2159  if self.config.usePtcGains:
2160  if ptcDataset is None:
2161  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2162  else:
2163  gain = ptcDataset.gain[amp.getName()]
2164  self.log.info("Using gain from Photon Transfer Curve.")
2165  else:
2166  gain = amp.getGain()
2167 
2168  if math.isnan(gain):
2169  gain = 1.0
2170  self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2171  elif gain <= 0:
2172  patchedGain = 1.0
2173  self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2174  amp.getName(), gain, patchedGain)
2175  gain = patchedGain
2176 
2177  if self.config.doEmpiricalReadNoise and overscanImage is None:
2178  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2179 
2180  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2181  stats = afwMath.StatisticsControl()
2182  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2183  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2184  self.log.info("Calculated empirical read noise for amp %s: %f.",
2185  amp.getName(), readNoise)
2186  elif self.config.usePtcReadNoise:
2187  if ptcDataset is None:
2188  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2189  else:
2190  readNoise = ptcDataset.noise[amp.getName()]
2191  self.log.info("Using read noise from Photon Transfer Curve.")
2192  else:
2193  readNoise = amp.getReadNoise()
2194 
2195  isrFunctions.updateVariance(
2196  maskedImage=ampExposure.getMaskedImage(),
2197  gain=gain,
2198  readNoise=readNoise,
2199  )
2200 
2201  def maskNegativeVariance(self, exposure):
2202  """Identify and mask pixels with negative variance values.
2203 
2204  Parameters
2205  ----------
2206  exposure : `lsst.afw.image.Exposure`
2207  Exposure to process.
2208 
2209  See Also
2210  --------
2211  lsst.ip.isr.isrFunctions.updateVariance
2212  """
2213  maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2214  bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2215  exposure.mask.array[bad] |= maskPlane
2216 
2217  def darkCorrection(self, exposure, darkExposure, invert=False):
2218  """Apply dark correction in place.
2219 
2220  Parameters
2221  ----------
2222  exposure : `lsst.afw.image.Exposure`
2223  Exposure to process.
2224  darkExposure : `lsst.afw.image.Exposure`
2225  Dark exposure of the same size as ``exposure``.
2226  invert : `Bool`, optional
2227  If True, re-add the dark to an already corrected image.
2228 
2229  Raises
2230  ------
2231  RuntimeError
2232  Raised if either ``exposure`` or ``darkExposure`` do not
2233  have their dark time defined.
2234 
2235  See Also
2236  --------
2237  lsst.ip.isr.isrFunctions.darkCorrection
2238  """
2239  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2240  if math.isnan(expScale):
2241  raise RuntimeError("Exposure darktime is NAN.")
2242  if darkExposure.getInfo().getVisitInfo() is not None \
2243  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2244  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2245  else:
2246  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2247  # so getDarkTime() does not exist.
2248  self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2249  darkScale = 1.0
2250 
2251  isrFunctions.darkCorrection(
2252  maskedImage=exposure.getMaskedImage(),
2253  darkMaskedImage=darkExposure.getMaskedImage(),
2254  expScale=expScale,
2255  darkScale=darkScale,
2256  invert=invert,
2257  trimToFit=self.config.doTrimToMatchCalib
2258  )
2259 
2260  def doLinearize(self, detector):
2261  """Check if linearization is needed for the detector cameraGeom.
2262 
2263  Checks config.doLinearize and the linearity type of the first
2264  amplifier.
2265 
2266  Parameters
2267  ----------
2268  detector : `lsst.afw.cameraGeom.Detector`
2269  Detector to get linearity type from.
2270 
2271  Returns
2272  -------
2273  doLinearize : `Bool`
2274  If True, linearization should be performed.
2275  """
2276  return self.config.doLinearize and \
2277  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2278 
2279  def flatCorrection(self, exposure, flatExposure, invert=False):
2280  """Apply flat correction in place.
2281 
2282  Parameters
2283  ----------
2284  exposure : `lsst.afw.image.Exposure`
2285  Exposure to process.
2286  flatExposure : `lsst.afw.image.Exposure`
2287  Flat exposure of the same size as ``exposure``.
2288  invert : `Bool`, optional
2289  If True, unflatten an already flattened image.
2290 
2291  See Also
2292  --------
2293  lsst.ip.isr.isrFunctions.flatCorrection
2294  """
2295  isrFunctions.flatCorrection(
2296  maskedImage=exposure.getMaskedImage(),
2297  flatMaskedImage=flatExposure.getMaskedImage(),
2298  scalingType=self.config.flatScalingType,
2299  userScale=self.config.flatUserScale,
2300  invert=invert,
2301  trimToFit=self.config.doTrimToMatchCalib
2302  )
2303 
2304  def saturationDetection(self, exposure, amp):
2305  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2306 
2307  Parameters
2308  ----------
2309  exposure : `lsst.afw.image.Exposure`
2310  Exposure to process. Only the amplifier DataSec is processed.
2311  amp : `lsst.afw.table.AmpInfoCatalog`
2312  Amplifier detector data.
2313 
2314  See Also
2315  --------
2316  lsst.ip.isr.isrFunctions.makeThresholdMask
2317  """
2318  if not math.isnan(amp.getSaturation()):
2319  maskedImage = exposure.getMaskedImage()
2320  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2321  isrFunctions.makeThresholdMask(
2322  maskedImage=dataView,
2323  threshold=amp.getSaturation(),
2324  growFootprints=0,
2325  maskName=self.config.saturatedMaskName,
2326  )
2327 
2328  def saturationInterpolation(self, exposure):
2329  """Interpolate over saturated pixels, in place.
2330 
2331  This method should be called after `saturationDetection`, to
2332  ensure that the saturated pixels have been identified in the
2333  SAT mask. It should also be called after `assembleCcd`, since
2334  saturated regions may cross amplifier boundaries.
2335 
2336  Parameters
2337  ----------
2338  exposure : `lsst.afw.image.Exposure`
2339  Exposure to process.
2340 
2341  See Also
2342  --------
2343  lsst.ip.isr.isrTask.saturationDetection
2344  lsst.ip.isr.isrFunctions.interpolateFromMask
2345  """
2346  isrFunctions.interpolateFromMask(
2347  maskedImage=exposure.getMaskedImage(),
2348  fwhm=self.config.fwhm,
2349  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2350  maskNameList=list(self.config.saturatedMaskName),
2351  )
2352 
2353  def suspectDetection(self, exposure, amp):
2354  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2355 
2356  Parameters
2357  ----------
2358  exposure : `lsst.afw.image.Exposure`
2359  Exposure to process. Only the amplifier DataSec is processed.
2360  amp : `lsst.afw.table.AmpInfoCatalog`
2361  Amplifier detector data.
2362 
2363  See Also
2364  --------
2365  lsst.ip.isr.isrFunctions.makeThresholdMask
2366 
2367  Notes
2368  -----
2369  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2370  This is intended to indicate pixels that may be affected by unknown systematics;
2371  for example if non-linearity corrections above a certain level are unstable
2372  then that would be a useful value for suspectLevel. A value of `nan` indicates
2373  that no such level exists and no pixels are to be masked as suspicious.
2374  """
2375  suspectLevel = amp.getSuspectLevel()
2376  if math.isnan(suspectLevel):
2377  return
2378 
2379  maskedImage = exposure.getMaskedImage()
2380  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2381  isrFunctions.makeThresholdMask(
2382  maskedImage=dataView,
2383  threshold=suspectLevel,
2384  growFootprints=0,
2385  maskName=self.config.suspectMaskName,
2386  )
2387 
2388  def maskDefect(self, exposure, defectBaseList):
2389  """Mask defects using mask plane "BAD", in place.
2390 
2391  Parameters
2392  ----------
2393  exposure : `lsst.afw.image.Exposure`
2394  Exposure to process.
2395  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2396  `lsst.afw.image.DefectBase`.
2397  List of defects to mask.
2398 
2399  Notes
2400  -----
2401  Call this after CCD assembly, since defects may cross amplifier boundaries.
2402  """
2403  maskedImage = exposure.getMaskedImage()
2404  if not isinstance(defectBaseList, Defects):
2405  # Promotes DefectBase to Defect
2406  defectList = Defects(defectBaseList)
2407  else:
2408  defectList = defectBaseList
2409  defectList.maskPixels(maskedImage, maskName="BAD")
2410 
2411  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2412  """Mask edge pixels with applicable mask plane.
2413 
2414  Parameters
2415  ----------
2416  exposure : `lsst.afw.image.Exposure`
2417  Exposure to process.
2418  numEdgePixels : `int`, optional
2419  Number of edge pixels to mask.
2420  maskPlane : `str`, optional
2421  Mask plane name to use.
2422  level : `str`, optional
2423  Level at which to mask edges.
2424  """
2425  maskedImage = exposure.getMaskedImage()
2426  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2427 
2428  if numEdgePixels > 0:
2429  if level == 'DETECTOR':
2430  boxes = [maskedImage.getBBox()]
2431  elif level == 'AMP':
2432  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2433 
2434  for box in boxes:
2435  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2436  subImage = maskedImage[box]
2437  box.grow(-numEdgePixels)
2438  # Mask pixels outside box
2439  SourceDetectionTask.setEdgeBits(
2440  subImage,
2441  box,
2442  maskBitMask)
2443 
2444  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2445  """Mask and interpolate defects using mask plane "BAD", in place.
2446 
2447  Parameters
2448  ----------
2449  exposure : `lsst.afw.image.Exposure`
2450  Exposure to process.
2451  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2452  `lsst.afw.image.DefectBase`.
2453  List of defects to mask and interpolate.
2454 
2455  See Also
2456  --------
2457  lsst.ip.isr.isrTask.maskDefect
2458  """
2459  self.maskDefectmaskDefect(exposure, defectBaseList)
2460  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2461  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2462  isrFunctions.interpolateFromMask(
2463  maskedImage=exposure.getMaskedImage(),
2464  fwhm=self.config.fwhm,
2465  growSaturatedFootprints=0,
2466  maskNameList=["BAD"],
2467  )
2468 
2469  def maskNan(self, exposure):
2470  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2471 
2472  Parameters
2473  ----------
2474  exposure : `lsst.afw.image.Exposure`
2475  Exposure to process.
2476 
2477  Notes
2478  -----
2479  We mask over all non-finite values (NaN, inf), including those
2480  that are masked with other bits (because those may or may not be
2481  interpolated over later, and we want to remove all NaN/infs).
2482  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2483  preserve the historical name.
2484  """
2485  maskedImage = exposure.getMaskedImage()
2486 
2487  # Find and mask NaNs
2488  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2489  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2490  numNans = maskNans(maskedImage, maskVal)
2491  self.metadata.set("NUMNANS", numNans)
2492  if numNans > 0:
2493  self.log.warning("There were %d unmasked NaNs.", numNans)
2494 
2495  def maskAndInterpolateNan(self, exposure):
2496  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2497  in place.
2498 
2499  Parameters
2500  ----------
2501  exposure : `lsst.afw.image.Exposure`
2502  Exposure to process.
2503 
2504  See Also
2505  --------
2506  lsst.ip.isr.isrTask.maskNan
2507  """
2508  self.maskNanmaskNan(exposure)
2509  isrFunctions.interpolateFromMask(
2510  maskedImage=exposure.getMaskedImage(),
2511  fwhm=self.config.fwhm,
2512  growSaturatedFootprints=0,
2513  maskNameList=["UNMASKEDNAN"],
2514  )
2515 
2516  def measureBackground(self, exposure, IsrQaConfig=None):
2517  """Measure the image background in subgrids, for quality control purposes.
2518 
2519  Parameters
2520  ----------
2521  exposure : `lsst.afw.image.Exposure`
2522  Exposure to process.
2523  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2524  Configuration object containing parameters on which background
2525  statistics and subgrids to use.
2526  """
2527  if IsrQaConfig is not None:
2528  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2529  IsrQaConfig.flatness.nIter)
2530  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2531  statsControl.setAndMask(maskVal)
2532  maskedImage = exposure.getMaskedImage()
2533  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2534  skyLevel = stats.getValue(afwMath.MEDIAN)
2535  skySigma = stats.getValue(afwMath.STDEVCLIP)
2536  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2537  metadata = exposure.getMetadata()
2538  metadata.set('SKYLEVEL', skyLevel)
2539  metadata.set('SKYSIGMA', skySigma)
2540 
2541  # calcluating flatlevel over the subgrids
2542  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2543  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2544  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2545  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2546  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2547  skyLevels = numpy.zeros((nX, nY))
2548 
2549  for j in range(nY):
2550  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2551  for i in range(nX):
2552  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2553 
2554  xLLC = xc - meshXHalf
2555  yLLC = yc - meshYHalf
2556  xURC = xc + meshXHalf - 1
2557  yURC = yc + meshYHalf - 1
2558 
2559  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2560  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2561 
2562  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2563 
2564  good = numpy.where(numpy.isfinite(skyLevels))
2565  skyMedian = numpy.median(skyLevels[good])
2566  flatness = (skyLevels[good] - skyMedian) / skyMedian
2567  flatness_rms = numpy.std(flatness)
2568  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2569 
2570  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2571  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2572  nX, nY, flatness_pp, flatness_rms)
2573 
2574  metadata.set('FLATNESS_PP', float(flatness_pp))
2575  metadata.set('FLATNESS_RMS', float(flatness_rms))
2576  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2577  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2578  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2579 
2580  def roughZeroPoint(self, exposure):
2581  """Set an approximate magnitude zero point for the exposure.
2582 
2583  Parameters
2584  ----------
2585  exposure : `lsst.afw.image.Exposure`
2586  Exposure to process.
2587  """
2588  filterLabel = exposure.getFilterLabel()
2589  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2590 
2591  if physicalFilter in self.config.fluxMag0T1:
2592  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2593  else:
2594  self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2595  fluxMag0 = self.config.defaultFluxMag0T1
2596 
2597  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2598  if not expTime > 0: # handle NaN as well as <= 0
2599  self.log.warning("Non-positive exposure time; skipping rough zero point.")
2600  return
2601 
2602  self.log.info("Setting rough magnitude zero point for filter %s: %f",
2603  physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2604  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2605 
2606  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2607  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2608 
2609  Parameters
2610  ----------
2611  ccdExposure : `lsst.afw.image.Exposure`
2612  Exposure to process.
2613  fpPolygon : `lsst.afw.geom.Polygon`
2614  Polygon in focal plane coordinates.
2615  """
2616  # Get ccd corners in focal plane coordinates
2617  ccd = ccdExposure.getDetector()
2618  fpCorners = ccd.getCorners(FOCAL_PLANE)
2619  ccdPolygon = Polygon(fpCorners)
2620 
2621  # Get intersection of ccd corners with fpPolygon
2622  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2623 
2624  # Transform back to pixel positions and build new polygon
2625  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2626  validPolygon = Polygon(ccdPoints)
2627  ccdExposure.getInfo().setValidPolygon(validPolygon)
2628 
2629  @contextmanager
2630  def flatContext(self, exp, flat, dark=None):
2631  """Context manager that applies and removes flats and darks,
2632  if the task is configured to apply them.
2633 
2634  Parameters
2635  ----------
2636  exp : `lsst.afw.image.Exposure`
2637  Exposure to process.
2638  flat : `lsst.afw.image.Exposure`
2639  Flat exposure the same size as ``exp``.
2640  dark : `lsst.afw.image.Exposure`, optional
2641  Dark exposure the same size as ``exp``.
2642 
2643  Yields
2644  ------
2645  exp : `lsst.afw.image.Exposure`
2646  The flat and dark corrected exposure.
2647  """
2648  if self.config.doDark and dark is not None:
2649  self.darkCorrectiondarkCorrection(exp, dark)
2650  if self.config.doFlat:
2651  self.flatCorrectionflatCorrection(exp, flat)
2652  try:
2653  yield exp
2654  finally:
2655  if self.config.doFlat:
2656  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2657  if self.config.doDark and dark is not None:
2658  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2659 
2660  def debugView(self, exposure, stepname):
2661  """Utility function to examine ISR exposure at different stages.
2662 
2663  Parameters
2664  ----------
2665  exposure : `lsst.afw.image.Exposure`
2666  Exposure to view.
2667  stepname : `str`
2668  State of processing to view.
2669  """
2670  frame = getDebugFrame(self._display, stepname)
2671  if frame:
2672  display = getDisplay(frame)
2673  display.scale('asinh', 'zscale')
2674  display.mtv(exposure)
2675  prompt = "Press Enter to continue [c]... "
2676  while True:
2677  ans = input(prompt).lower()
2678  if ans in ("", "c",):
2679  break
2680 
2681 
2682 class FakeAmp(object):
2683  """A Detector-like object that supports returning gain and saturation level
2684 
2685  This is used when the input exposure does not have a detector.
2686 
2687  Parameters
2688  ----------
2689  exposure : `lsst.afw.image.Exposure`
2690  Exposure to generate a fake amplifier for.
2691  config : `lsst.ip.isr.isrTaskConfig`
2692  Configuration to apply to the fake amplifier.
2693  """
2694 
2695  def __init__(self, exposure, config):
2696  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2697  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2698  self._gain_gain = config.gain
2699  self._readNoise_readNoise = config.readNoise
2700  self._saturation_saturation = config.saturation
2701 
2702  def getBBox(self):
2703  return self._bbox_bbox
2704 
2705  def getRawBBox(self):
2706  return self._bbox_bbox
2707 
2709  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2710 
2711  def getGain(self):
2712  return self._gain_gain
2713 
2714  def getReadNoise(self):
2715  return self._readNoise_readNoise
2716 
2717  def getSaturation(self):
2718  return self._saturation_saturation
2719 
2720  def getSuspectLevel(self):
2721  return float("NaN")
2722 
2723 
2724 class RunIsrConfig(pexConfig.Config):
2725  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2726 
2727 
2728 class RunIsrTask(pipeBase.CmdLineTask):
2729  """Task to wrap the default IsrTask to allow it to be retargeted.
2730 
2731  The standard IsrTask can be called directly from a command line
2732  program, but doing so removes the ability of the task to be
2733  retargeted. As most cameras override some set of the IsrTask
2734  methods, this would remove those data-specific methods in the
2735  output post-ISR images. This wrapping class fixes the issue,
2736  allowing identical post-ISR images to be generated by both the
2737  processCcd and isrTask code.
2738  """
2739  ConfigClass = RunIsrConfig
2740  _DefaultName = "runIsr"
2741 
2742  def __init__(self, *args, **kwargs):
2743  super().__init__(*args, **kwargs)
2744  self.makeSubtask("isr")
2745 
2746  def runDataRef(self, dataRef):
2747  """
2748  Parameters
2749  ----------
2750  dataRef : `lsst.daf.persistence.ButlerDataRef`
2751  data reference of the detector data to be processed
2752 
2753  Returns
2754  -------
2755  result : `pipeBase.Struct`
2756  Result struct with component:
2757 
2758  - exposure : `lsst.afw.image.Exposure`
2759  Post-ISR processed exposure.
2760  """
2761  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2708
def __init__(self, exposure, config)
Definition: isrTask.py:2695
def __init__(self, *config=None)
Definition: isrTask.py:273
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2279
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2495
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2328
def runDataRef(self, sensorRef)
Definition: isrTask.py:1743
def maskNan(self, exposure)
Definition: isrTask.py:2469
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1927
def debugView(self, exposure, stepname)
Definition: isrTask.py:2660
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1794
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:2201
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2304
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2388
def __init__(self, **kwargs)
Definition: isrTask.py:961
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:971
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2411
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1998
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2516
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2580
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2444
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2606
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1069
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1842
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1251
def doLinearize(self, detector)
Definition: isrTask.py:2260
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2630
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1890
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2353
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2126
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2217
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2742
def runDataRef(self, dataRef)
Definition: isrTask.py:2746
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35