lsst.ip.isr  22.0.0-12-ga88477b+cb4f42a696
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 from .defects import Defects
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  findFirst=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  minimum=0, # can fall back to cameraGeom
123  )
124  # TODO: DM-25348. This does not work yet to correctly load
125  # possible crosstalk sources.
126  crosstalkSources = cT.PrerequisiteInput(
127  name="isrOverscanCorrected",
128  doc="Overscan corrected input images.",
129  storageClass="Exposure",
130  dimensions=["instrument", "exposure", "detector"],
131  deferLoad=True,
132  multiple=True,
133  lookupFunction=crosstalkSourceLookup,
134  minimum=0, # not needed for all instruments, no config to control this
135  )
136  bias = cT.PrerequisiteInput(
137  name="bias",
138  doc="Input bias calibration.",
139  storageClass="ExposureF",
140  dimensions=["instrument", "detector"],
141  isCalibration=True,
142  )
143  dark = cT.PrerequisiteInput(
144  name='dark',
145  doc="Input dark calibration.",
146  storageClass="ExposureF",
147  dimensions=["instrument", "detector"],
148  isCalibration=True,
149  )
150  flat = cT.PrerequisiteInput(
151  name="flat",
152  doc="Input flat calibration.",
153  storageClass="ExposureF",
154  dimensions=["instrument", "physical_filter", "detector"],
155  isCalibration=True,
156  )
157  ptc = cT.PrerequisiteInput(
158  name="ptc",
159  doc="Input Photon Transfer Curve dataset",
160  storageClass="PhotonTransferCurveDataset",
161  dimensions=["instrument", "detector"],
162  isCalibration=True,
163  )
164  fringes = cT.PrerequisiteInput(
165  name="fringe",
166  doc="Input fringe calibration.",
167  storageClass="ExposureF",
168  dimensions=["instrument", "physical_filter", "detector"],
169  isCalibration=True,
170  minimum=0, # only needed for some bands, even when enabled
171  )
172  strayLightData = cT.PrerequisiteInput(
173  name='yBackground',
174  doc="Input stray light calibration.",
175  storageClass="StrayLightData",
176  dimensions=["instrument", "physical_filter", "detector"],
177  isCalibration=True,
178  minimum=0, # only needed for some bands, even when enabled
179  )
180  bfKernel = cT.PrerequisiteInput(
181  name='bfKernel',
182  doc="Input brighter-fatter kernel.",
183  storageClass="NumpyArray",
184  dimensions=["instrument"],
185  isCalibration=True,
186  minimum=0, # can use either bfKernel or newBFKernel
187  )
188  newBFKernel = cT.PrerequisiteInput(
189  name='brighterFatterKernel',
190  doc="Newer complete kernel + gain solutions.",
191  storageClass="BrighterFatterKernel",
192  dimensions=["instrument", "detector"],
193  isCalibration=True,
194  minimum=0, # can use either bfKernel or newBFKernel
195  )
196  defects = cT.PrerequisiteInput(
197  name='defects',
198  doc="Input defect tables.",
199  storageClass="Defects",
200  dimensions=["instrument", "detector"],
201  isCalibration=True,
202  )
203  linearizer = cT.PrerequisiteInput(
204  name='linearizer',
205  storageClass="Linearizer",
206  doc="Linearity correction calibration.",
207  dimensions=["instrument", "detector"],
208  isCalibration=True,
209  minimum=0, # can fall back to cameraGeom
210  )
211  opticsTransmission = cT.PrerequisiteInput(
212  name="transmission_optics",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the optics.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  filterTransmission = cT.PrerequisiteInput(
219  name="transmission_filter",
220  storageClass="TransmissionCurve",
221  doc="Transmission curve due to the filter.",
222  dimensions=["instrument", "physical_filter"],
223  isCalibration=True,
224  )
225  sensorTransmission = cT.PrerequisiteInput(
226  name="transmission_sensor",
227  storageClass="TransmissionCurve",
228  doc="Transmission curve due to the sensor.",
229  dimensions=["instrument", "detector"],
230  isCalibration=True,
231  )
232  atmosphereTransmission = cT.PrerequisiteInput(
233  name="transmission_atmosphere",
234  storageClass="TransmissionCurve",
235  doc="Transmission curve due to the atmosphere.",
236  dimensions=["instrument"],
237  isCalibration=True,
238  )
239  illumMaskedImage = cT.PrerequisiteInput(
240  name="illum",
241  doc="Input illumination correction.",
242  storageClass="MaskedImageF",
243  dimensions=["instrument", "physical_filter", "detector"],
244  isCalibration=True,
245  )
246 
247  outputExposure = cT.Output(
248  name='postISRCCD',
249  doc="Output ISR processed exposure.",
250  storageClass="Exposure",
251  dimensions=["instrument", "exposure", "detector"],
252  )
253  preInterpExposure = cT.Output(
254  name='preInterpISRCCD',
255  doc="Output ISR processed exposure, with pixels left uninterpolated.",
256  storageClass="ExposureF",
257  dimensions=["instrument", "exposure", "detector"],
258  )
259  outputOssThumbnail = cT.Output(
260  name="OssThumb",
261  doc="Output Overscan-subtracted thumbnail image.",
262  storageClass="Thumbnail",
263  dimensions=["instrument", "exposure", "detector"],
264  )
265  outputFlattenedThumbnail = cT.Output(
266  name="FlattenedThumb",
267  doc="Output flat-corrected thumbnail image.",
268  storageClass="Thumbnail",
269  dimensions=["instrument", "exposure", "detector"],
270  )
271 
272  def __init__(self, *, config=None):
273  super().__init__(config=config)
274 
275  if config.doBias is not True:
276  self.prerequisiteInputs.discard("bias")
277  if config.doLinearize is not True:
278  self.prerequisiteInputs.discard("linearizer")
279  if config.doCrosstalk is not True:
280  self.inputs.discard("crosstalkSources")
281  self.prerequisiteInputs.discard("crosstalk")
282  if config.doBrighterFatter is not True:
283  self.prerequisiteInputs.discard("bfKernel")
284  self.prerequisiteInputs.discard("newBFKernel")
285  if config.doDefect is not True:
286  self.prerequisiteInputs.discard("defects")
287  if config.doDark is not True:
288  self.prerequisiteInputs.discard("dark")
289  if config.doFlat is not True:
290  self.prerequisiteInputs.discard("flat")
291  if config.doFringe is not True:
292  self.prerequisiteInputs.discard("fringe")
293  if config.doStrayLight is not True:
294  self.prerequisiteInputs.discard("strayLightData")
295  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296  self.prerequisiteInputs.discard("ptc")
297  if config.doAttachTransmissionCurve is not True:
298  self.prerequisiteInputs.discard("opticsTransmission")
299  self.prerequisiteInputs.discard("filterTransmission")
300  self.prerequisiteInputs.discard("sensorTransmission")
301  self.prerequisiteInputs.discard("atmosphereTransmission")
302  if config.doUseOpticsTransmission is not True:
303  self.prerequisiteInputs.discard("opticsTransmission")
304  if config.doUseFilterTransmission is not True:
305  self.prerequisiteInputs.discard("filterTransmission")
306  if config.doUseSensorTransmission is not True:
307  self.prerequisiteInputs.discard("sensorTransmission")
308  if config.doUseAtmosphereTransmission is not True:
309  self.prerequisiteInputs.discard("atmosphereTransmission")
310  if config.doIlluminationCorrection is not True:
311  self.prerequisiteInputs.discard("illumMaskedImage")
312 
313  if config.doWrite is not True:
314  self.outputs.discard("outputExposure")
315  self.outputs.discard("preInterpExposure")
316  self.outputs.discard("outputFlattenedThumbnail")
317  self.outputs.discard("outputOssThumbnail")
318  if config.doSaveInterpPixels is not True:
319  self.outputs.discard("preInterpExposure")
320  if config.qa.doThumbnailOss is not True:
321  self.outputs.discard("outputOssThumbnail")
322  if config.qa.doThumbnailFlattened is not True:
323  self.outputs.discard("outputFlattenedThumbnail")
324 
325 
326 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327  pipelineConnections=IsrTaskConnections):
328  """Configuration parameters for IsrTask.
329 
330  Items are grouped in the order in which they are executed by the task.
331  """
332  datasetType = pexConfig.Field(
333  dtype=str,
334  doc="Dataset type for input data; users will typically leave this alone, "
335  "but camera-specific ISR tasks will override it",
336  default="raw",
337  )
338 
339  fallbackFilterName = pexConfig.Field(
340  dtype=str,
341  doc="Fallback default filter name for calibrations.",
342  optional=True
343  )
344  useFallbackDate = pexConfig.Field(
345  dtype=bool,
346  doc="Pass observation date when using fallback filter.",
347  default=False,
348  )
349  expectWcs = pexConfig.Field(
350  dtype=bool,
351  default=True,
352  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353  )
354  fwhm = pexConfig.Field(
355  dtype=float,
356  doc="FWHM of PSF in arcseconds.",
357  default=1.0,
358  )
359  qa = pexConfig.ConfigField(
360  dtype=isrQa.IsrQaConfig,
361  doc="QA related configuration options.",
362  )
363 
364  # Image conversion configuration
365  doConvertIntToFloat = pexConfig.Field(
366  dtype=bool,
367  doc="Convert integer raw images to floating point values?",
368  default=True,
369  )
370 
371  # Saturated pixel handling.
372  doSaturation = pexConfig.Field(
373  dtype=bool,
374  doc="Mask saturated pixels? NB: this is totally independent of the"
375  " interpolation option - this is ONLY setting the bits in the mask."
376  " To have them interpolated make sure doSaturationInterpolation=True",
377  default=True,
378  )
379  saturatedMaskName = pexConfig.Field(
380  dtype=str,
381  doc="Name of mask plane to use in saturation detection and interpolation",
382  default="SAT",
383  )
384  saturation = pexConfig.Field(
385  dtype=float,
386  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387  default=float("NaN"),
388  )
389  growSaturationFootprintSize = pexConfig.Field(
390  dtype=int,
391  doc="Number of pixels by which to grow the saturation footprints",
392  default=1,
393  )
394 
395  # Suspect pixel handling.
396  doSuspect = pexConfig.Field(
397  dtype=bool,
398  doc="Mask suspect pixels?",
399  default=False,
400  )
401  suspectMaskName = pexConfig.Field(
402  dtype=str,
403  doc="Name of mask plane to use for suspect pixels",
404  default="SUSPECT",
405  )
406  numEdgeSuspect = pexConfig.Field(
407  dtype=int,
408  doc="Number of edge pixels to be flagged as untrustworthy.",
409  default=0,
410  )
411  edgeMaskLevel = pexConfig.ChoiceField(
412  dtype=str,
413  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414  default="DETECTOR",
415  allowed={
416  'DETECTOR': 'Mask only the edges of the full detector.',
417  'AMP': 'Mask edges of each amplifier.',
418  },
419  )
420 
421  # Initial masking options.
422  doSetBadRegions = pexConfig.Field(
423  dtype=bool,
424  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425  default=True,
426  )
427  badStatistic = pexConfig.ChoiceField(
428  dtype=str,
429  doc="How to estimate the average value for BAD regions.",
430  default='MEANCLIP',
431  allowed={
432  "MEANCLIP": "Correct using the (clipped) mean of good data",
433  "MEDIAN": "Correct using the median of the good data",
434  },
435  )
436 
437  # Overscan subtraction configuration.
438  doOverscan = pexConfig.Field(
439  dtype=bool,
440  doc="Do overscan subtraction?",
441  default=True,
442  )
443  overscan = pexConfig.ConfigurableField(
444  target=OverscanCorrectionTask,
445  doc="Overscan subtraction task for image segments.",
446  )
447 
448  overscanFitType = pexConfig.ChoiceField(
449  dtype=str,
450  doc="The method for fitting the overscan bias level.",
451  default='MEDIAN',
452  allowed={
453  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
454  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
455  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
456  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
457  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
458  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
459  "MEAN": "Correct using the mean of the overscan region",
460  "MEANCLIP": "Correct using a clipped mean of the overscan region",
461  "MEDIAN": "Correct using the median of the overscan region",
462  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
463  },
464  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
465  " This option will no longer be used, and will be removed after v20.")
466  )
467  overscanOrder = pexConfig.Field(
468  dtype=int,
469  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
470  "or number of spline knots if overscan fit type is a spline."),
471  default=1,
472  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
473  " This option will no longer be used, and will be removed after v20.")
474  )
475  overscanNumSigmaClip = pexConfig.Field(
476  dtype=float,
477  doc="Rejection threshold (sigma) for collapsing overscan before fit",
478  default=3.0,
479  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
480  " This option will no longer be used, and will be removed after v20.")
481  )
482  overscanIsInt = pexConfig.Field(
483  dtype=bool,
484  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485  " and overscan.FitType=MEDIAN_PER_ROW.",
486  default=True,
487  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
488  " This option will no longer be used, and will be removed after v20.")
489  )
490  # These options do not get deprecated, as they define how we slice up the image data.
491  overscanNumLeadingColumnsToSkip = pexConfig.Field(
492  dtype=int,
493  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494  default=0,
495  )
496  overscanNumTrailingColumnsToSkip = pexConfig.Field(
497  dtype=int,
498  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499  default=0,
500  )
501  overscanMaxDev = pexConfig.Field(
502  dtype=float,
503  doc="Maximum deviation from the median for overscan",
504  default=1000.0, check=lambda x: x > 0
505  )
506  overscanBiasJump = pexConfig.Field(
507  dtype=bool,
508  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509  default=False,
510  )
511  overscanBiasJumpKeyword = pexConfig.Field(
512  dtype=str,
513  doc="Header keyword containing information about devices.",
514  default="NO_SUCH_KEY",
515  )
516  overscanBiasJumpDevices = pexConfig.ListField(
517  dtype=str,
518  doc="List of devices that need piecewise overscan correction.",
519  default=(),
520  )
521  overscanBiasJumpLocation = pexConfig.Field(
522  dtype=int,
523  doc="Location of bias jump along y-axis.",
524  default=0,
525  )
526 
527  # Amplifier to CCD assembly configuration
528  doAssembleCcd = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Assemble amp-level exposures into a ccd-level exposure?"
532  )
533  assembleCcd = pexConfig.ConfigurableField(
534  target=AssembleCcdTask,
535  doc="CCD assembly task",
536  )
537 
538  # General calibration configuration.
539  doAssembleIsrExposures = pexConfig.Field(
540  dtype=bool,
541  default=False,
542  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543  )
544  doTrimToMatchCalib = pexConfig.Field(
545  dtype=bool,
546  default=False,
547  doc="Trim raw data to match calibration bounding boxes?"
548  )
549 
550  # Bias subtraction.
551  doBias = pexConfig.Field(
552  dtype=bool,
553  doc="Apply bias frame correction?",
554  default=True,
555  )
556  biasDataProductName = pexConfig.Field(
557  dtype=str,
558  doc="Name of the bias data product",
559  default="bias",
560  )
561  doBiasBeforeOverscan = pexConfig.Field(
562  dtype=bool,
563  doc="Reverse order of overscan and bias correction.",
564  default=False
565  )
566 
567  # Variance construction
568  doVariance = pexConfig.Field(
569  dtype=bool,
570  doc="Calculate variance?",
571  default=True
572  )
573  gain = pexConfig.Field(
574  dtype=float,
575  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576  default=float("NaN"),
577  )
578  readNoise = pexConfig.Field(
579  dtype=float,
580  doc="The read noise to use if no Detector is present in the Exposure",
581  default=0.0,
582  )
583  doEmpiricalReadNoise = pexConfig.Field(
584  dtype=bool,
585  default=False,
586  doc="Calculate empirical read noise instead of value from AmpInfo data?"
587  )
588  usePtcReadNoise = pexConfig.Field(
589  dtype=bool,
590  default=False,
591  doc="Use readnoise values from the Photon Transfer Curve?"
592  )
593  # Linearization.
594  doLinearize = pexConfig.Field(
595  dtype=bool,
596  doc="Correct for nonlinearity of the detector's response?",
597  default=True,
598  )
599 
600  # Crosstalk.
601  doCrosstalk = pexConfig.Field(
602  dtype=bool,
603  doc="Apply intra-CCD crosstalk correction?",
604  default=False,
605  )
606  doCrosstalkBeforeAssemble = pexConfig.Field(
607  dtype=bool,
608  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
609  default=False,
610  )
611  crosstalk = pexConfig.ConfigurableField(
612  target=CrosstalkTask,
613  doc="Intra-CCD crosstalk correction",
614  )
615 
616  # Masking options.
617  doDefect = pexConfig.Field(
618  dtype=bool,
619  doc="Apply correction for CCD defects, e.g. hot pixels?",
620  default=True,
621  )
622  doNanMasking = pexConfig.Field(
623  dtype=bool,
624  doc="Mask non-finite (NAN, inf) pixels?",
625  default=True,
626  )
627  doWidenSaturationTrails = pexConfig.Field(
628  dtype=bool,
629  doc="Widen bleed trails based on their width?",
630  default=True
631  )
632 
633  # Brighter-Fatter correction.
634  doBrighterFatter = pexConfig.Field(
635  dtype=bool,
636  default=False,
637  doc="Apply the brighter-fatter correction?"
638  )
639  brighterFatterLevel = pexConfig.ChoiceField(
640  dtype=str,
641  default="DETECTOR",
642  doc="The level at which to correct for brighter-fatter.",
643  allowed={
644  "AMP": "Every amplifier treated separately.",
645  "DETECTOR": "One kernel per detector",
646  }
647  )
648  brighterFatterMaxIter = pexConfig.Field(
649  dtype=int,
650  default=10,
651  doc="Maximum number of iterations for the brighter-fatter correction"
652  )
653  brighterFatterThreshold = pexConfig.Field(
654  dtype=float,
655  default=1000,
656  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
657  "absolute value of the difference between the current corrected image and the one "
658  "from the previous iteration summed over all the pixels."
659  )
660  brighterFatterApplyGain = pexConfig.Field(
661  dtype=bool,
662  default=True,
663  doc="Should the gain be applied when applying the brighter-fatter correction?"
664  )
665  brighterFatterMaskListToInterpolate = pexConfig.ListField(
666  dtype=str,
667  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
668  "correction.",
669  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
670  )
671  brighterFatterMaskGrowSize = pexConfig.Field(
672  dtype=int,
673  default=0,
674  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
675  "when brighter-fatter correction is applied."
676  )
677 
678  # Dark subtraction.
679  doDark = pexConfig.Field(
680  dtype=bool,
681  doc="Apply dark frame correction?",
682  default=True,
683  )
684  darkDataProductName = pexConfig.Field(
685  dtype=str,
686  doc="Name of the dark data product",
687  default="dark",
688  )
689 
690  # Camera-specific stray light removal.
691  doStrayLight = pexConfig.Field(
692  dtype=bool,
693  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
694  default=False,
695  )
696  strayLight = pexConfig.ConfigurableField(
697  target=StrayLightTask,
698  doc="y-band stray light correction"
699  )
700 
701  # Flat correction.
702  doFlat = pexConfig.Field(
703  dtype=bool,
704  doc="Apply flat field correction?",
705  default=True,
706  )
707  flatDataProductName = pexConfig.Field(
708  dtype=str,
709  doc="Name of the flat data product",
710  default="flat",
711  )
712  flatScalingType = pexConfig.ChoiceField(
713  dtype=str,
714  doc="The method for scaling the flat on the fly.",
715  default='USER',
716  allowed={
717  "USER": "Scale by flatUserScale",
718  "MEAN": "Scale by the inverse of the mean",
719  "MEDIAN": "Scale by the inverse of the median",
720  },
721  )
722  flatUserScale = pexConfig.Field(
723  dtype=float,
724  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
725  default=1.0,
726  )
727  doTweakFlat = pexConfig.Field(
728  dtype=bool,
729  doc="Tweak flats to match observed amplifier ratios?",
730  default=False
731  )
732 
733  # Amplifier normalization based on gains instead of using flats configuration.
734  doApplyGains = pexConfig.Field(
735  dtype=bool,
736  doc="Correct the amplifiers for their gains instead of applying flat correction",
737  default=False,
738  )
739  usePtcGains = pexConfig.Field(
740  dtype=bool,
741  doc="Use the gain values from the Photon Transfer Curve?",
742  default=False,
743  )
744  normalizeGains = pexConfig.Field(
745  dtype=bool,
746  doc="Normalize all the amplifiers in each CCD to have the same median value.",
747  default=False,
748  )
749 
750  # Fringe correction.
751  doFringe = pexConfig.Field(
752  dtype=bool,
753  doc="Apply fringe correction?",
754  default=True,
755  )
756  fringe = pexConfig.ConfigurableField(
757  target=FringeTask,
758  doc="Fringe subtraction task",
759  )
760  fringeAfterFlat = pexConfig.Field(
761  dtype=bool,
762  doc="Do fringe subtraction after flat-fielding?",
763  default=True,
764  )
765 
766  # Initial CCD-level background statistics options.
767  doMeasureBackground = pexConfig.Field(
768  dtype=bool,
769  doc="Measure the background level on the reduced image?",
770  default=False,
771  )
772 
773  # Camera-specific masking configuration.
774  doCameraSpecificMasking = pexConfig.Field(
775  dtype=bool,
776  doc="Mask camera-specific bad regions?",
777  default=False,
778  )
779  masking = pexConfig.ConfigurableField(
780  target=MaskingTask,
781  doc="Masking task."
782  )
783 
784  # Interpolation options.
785 
786  doInterpolate = pexConfig.Field(
787  dtype=bool,
788  doc="Interpolate masked pixels?",
789  default=True,
790  )
791  doSaturationInterpolation = pexConfig.Field(
792  dtype=bool,
793  doc="Perform interpolation over pixels masked as saturated?"
794  " NB: This is independent of doSaturation; if that is False this plane"
795  " will likely be blank, resulting in a no-op here.",
796  default=True,
797  )
798  doNanInterpolation = pexConfig.Field(
799  dtype=bool,
800  doc="Perform interpolation over pixels masked as NaN?"
801  " NB: This is independent of doNanMasking; if that is False this plane"
802  " will likely be blank, resulting in a no-op here.",
803  default=True,
804  )
805  doNanInterpAfterFlat = pexConfig.Field(
806  dtype=bool,
807  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
808  "also have to interpolate them before flat-fielding."),
809  default=False,
810  )
811  maskListToInterpolate = pexConfig.ListField(
812  dtype=str,
813  doc="List of mask planes that should be interpolated.",
814  default=['SAT', 'BAD'],
815  )
816  doSaveInterpPixels = pexConfig.Field(
817  dtype=bool,
818  doc="Save a copy of the pre-interpolated pixel values?",
819  default=False,
820  )
821 
822  # Default photometric calibration options.
823  fluxMag0T1 = pexConfig.DictField(
824  keytype=str,
825  itemtype=float,
826  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
827  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
828  ))
829  )
830  defaultFluxMag0T1 = pexConfig.Field(
831  dtype=float,
832  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
833  default=pow(10.0, 0.4*28.0)
834  )
835 
836  # Vignette correction configuration.
837  doVignette = pexConfig.Field(
838  dtype=bool,
839  doc="Apply vignetting parameters?",
840  default=False,
841  )
842  vignette = pexConfig.ConfigurableField(
843  target=VignetteTask,
844  doc="Vignetting task.",
845  )
846 
847  # Transmission curve configuration.
848  doAttachTransmissionCurve = pexConfig.Field(
849  dtype=bool,
850  default=False,
851  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
852  )
853  doUseOpticsTransmission = pexConfig.Field(
854  dtype=bool,
855  default=True,
856  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
857  )
858  doUseFilterTransmission = pexConfig.Field(
859  dtype=bool,
860  default=True,
861  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
862  )
863  doUseSensorTransmission = pexConfig.Field(
864  dtype=bool,
865  default=True,
866  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
867  )
868  doUseAtmosphereTransmission = pexConfig.Field(
869  dtype=bool,
870  default=True,
871  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
872  )
873 
874  # Illumination correction.
875  doIlluminationCorrection = pexConfig.Field(
876  dtype=bool,
877  default=False,
878  doc="Perform illumination correction?"
879  )
880  illuminationCorrectionDataProductName = pexConfig.Field(
881  dtype=str,
882  doc="Name of the illumination correction data product.",
883  default="illumcor",
884  )
885  illumScale = pexConfig.Field(
886  dtype=float,
887  doc="Scale factor for the illumination correction.",
888  default=1.0,
889  )
890  illumFilters = pexConfig.ListField(
891  dtype=str,
892  default=[],
893  doc="Only perform illumination correction for these filters."
894  )
895 
896  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
897  doWrite = pexConfig.Field(
898  dtype=bool,
899  doc="Persist postISRCCD?",
900  default=True,
901  )
902 
903  def validate(self):
904  super().validate()
905  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
906  raise ValueError("You may not specify both doFlat and doApplyGains")
907  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
908  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
909  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
910  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
911  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
912  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
913  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
914  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
915 
916 
917 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
918  """Apply common instrument signature correction algorithms to a raw frame.
919 
920  The process for correcting imaging data is very similar from
921  camera to camera. This task provides a vanilla implementation of
922  doing these corrections, including the ability to turn certain
923  corrections off if they are not needed. The inputs to the primary
924  method, `run()`, are a raw exposure to be corrected and the
925  calibration data products. The raw input is a single chip sized
926  mosaic of all amps including overscans and other non-science
927  pixels. The method `runDataRef()` identifies and defines the
928  calibration data products, and is intended for use by a
929  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
930  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
931  subclassed for different camera, although the most camera specific
932  methods have been split into subtasks that can be redirected
933  appropriately.
934 
935  The __init__ method sets up the subtasks for ISR processing, using
936  the defaults from `lsst.ip.isr`.
937 
938  Parameters
939  ----------
940  args : `list`
941  Positional arguments passed to the Task constructor. None used at this time.
942  kwargs : `dict`, optional
943  Keyword arguments passed on to the Task constructor. None used at this time.
944  """
945  ConfigClass = IsrTaskConfig
946  _DefaultName = "isr"
947 
948  def __init__(self, **kwargs):
949  super().__init__(**kwargs)
950  self.makeSubtask("assembleCcd")
951  self.makeSubtask("crosstalk")
952  self.makeSubtask("strayLight")
953  self.makeSubtask("fringe")
954  self.makeSubtask("masking")
955  self.makeSubtask("overscan")
956  self.makeSubtask("vignette")
957 
958  def runQuantum(self, butlerQC, inputRefs, outputRefs):
959  inputs = butlerQC.get(inputRefs)
960 
961  try:
962  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
963  except Exception as e:
964  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
965  (inputRefs, e))
966 
967  inputs['isGen3'] = True
968 
969  detector = inputs['ccdExposure'].getDetector()
970 
971  if self.config.doCrosstalk is True:
972  # Crosstalk sources need to be defined by the pipeline
973  # yaml if they exist.
974  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
975  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
976  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
977  else:
978  coeffVector = (self.config.crosstalk.crosstalkValues
979  if self.config.crosstalk.useConfigCoefficients else None)
980  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
981  inputs['crosstalk'] = crosstalkCalib
982  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
983  if 'crosstalkSources' not in inputs:
984  self.log.warning("No crosstalkSources found for chip with interChip terms!")
985 
986  if self.doLinearizedoLinearize(detector) is True:
987  if 'linearizer' in inputs:
988  if isinstance(inputs['linearizer'], dict):
989  linearizer = linearize.Linearizer(detector=detector, log=self.log)
990  linearizer.fromYaml(inputs['linearizer'])
991  self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
992  elif isinstance(inputs['linearizer'], numpy.ndarray):
993  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
994  detector=detector,
995  log=self.log)
996  self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
997  else:
998  linearizer = inputs['linearizer']
999  linearizer.log = self.log
1000  inputs['linearizer'] = linearizer
1001  else:
1002  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1003  self.log.warning("Constructing linearizer from cameraGeom information.")
1004 
1005  if self.config.doDefect is True:
1006  if "defects" in inputs and inputs['defects'] is not None:
1007  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1008  # masking expects a list of defects defined by their bounding box
1009  if not isinstance(inputs["defects"], Defects):
1010  inputs["defects"] = Defects.fromTable(inputs["defects"])
1011 
1012  # Load the correct style of brighter-fatter kernel, and repack
1013  # the information as a numpy array.
1014  if self.config.doBrighterFatter:
1015  brighterFatterKernel = inputs.pop('newBFKernel', None)
1016  if brighterFatterKernel is None:
1017  brighterFatterKernel = inputs.get('bfKernel', None)
1018 
1019  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1020  # This is a ISR calib kernel
1021  detName = detector.getName()
1022  level = brighterFatterKernel.level
1023 
1024  # This is expected to be a dictionary of amp-wise gains.
1025  inputs['bfGains'] = brighterFatterKernel.gain
1026  if self.config.brighterFatterLevel == 'DETECTOR':
1027  if level == 'DETECTOR':
1028  if detName in brighterFatterKernel.detKernels:
1029  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1030  else:
1031  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1032  elif level == 'AMP':
1033  self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1034  "fatter kernels.")
1035  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1036  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1037  elif self.config.brighterFatterLevel == 'AMP':
1038  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1039 
1040  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1041  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1042  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1043  expId=expId,
1044  assembler=self.assembleCcd
1045  if self.config.doAssembleIsrExposures else None)
1046  else:
1047  inputs['fringes'] = pipeBase.Struct(fringes=None)
1048 
1049  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1050  if 'strayLightData' not in inputs:
1051  inputs['strayLightData'] = None
1052 
1053  outputs = self.runrun(**inputs)
1054  butlerQC.put(outputs, outputRefs)
1055 
1056  def readIsrData(self, dataRef, rawExposure):
1057  """Retrieve necessary frames for instrument signature removal.
1058 
1059  Pre-fetching all required ISR data products limits the IO
1060  required by the ISR. Any conflict between the calibration data
1061  available and that needed for ISR is also detected prior to
1062  doing processing, allowing it to fail quickly.
1063 
1064  Parameters
1065  ----------
1066  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1067  Butler reference of the detector data to be processed
1068  rawExposure : `afw.image.Exposure`
1069  The raw exposure that will later be corrected with the
1070  retrieved calibration data; should not be modified in this
1071  method.
1072 
1073  Returns
1074  -------
1075  result : `lsst.pipe.base.Struct`
1076  Result struct with components (which may be `None`):
1077  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1078  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1079  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1080  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1081  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1082  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1083  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1084  - ``fringes``: `lsst.pipe.base.Struct` with components:
1085  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1086  - ``seed``: random seed derived from the ccdExposureId for random
1087  number generator (`uint32`).
1088  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1089  A ``TransmissionCurve`` that represents the throughput of the optics,
1090  to be evaluated in focal-plane coordinates.
1091  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1092  A ``TransmissionCurve`` that represents the throughput of the filter
1093  itself, to be evaluated in focal-plane coordinates.
1094  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1095  A ``TransmissionCurve`` that represents the throughput of the sensor
1096  itself, to be evaluated in post-assembly trimmed detector coordinates.
1097  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1098  A ``TransmissionCurve`` that represents the throughput of the
1099  atmosphere, assumed to be spatially constant.
1100  - ``strayLightData`` : `object`
1101  An opaque object containing calibration information for
1102  stray-light correction. If `None`, no correction will be
1103  performed.
1104  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1105 
1106  Raises
1107  ------
1108  NotImplementedError :
1109  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1110  """
1111  try:
1112  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1113  dateObs = dateObs.toPython().isoformat()
1114  except RuntimeError:
1115  self.log.warning("Unable to identify dateObs for rawExposure.")
1116  dateObs = None
1117 
1118  ccd = rawExposure.getDetector()
1119  filterLabel = rawExposure.getFilterLabel()
1120  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1121  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1122  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1123  if self.config.doBias else None)
1124  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1125  linearizer = (dataRef.get("linearizer", immediate=True)
1126  if self.doLinearizedoLinearize(ccd) else None)
1127  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1128  linearizer.log = self.log
1129  if isinstance(linearizer, numpy.ndarray):
1130  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1131 
1132  crosstalkCalib = None
1133  if self.config.doCrosstalk:
1134  try:
1135  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1136  except NoResults:
1137  coeffVector = (self.config.crosstalk.crosstalkValues
1138  if self.config.crosstalk.useConfigCoefficients else None)
1139  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1140  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1141  if self.config.doCrosstalk else None)
1142 
1143  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1144  if self.config.doDark else None)
1145  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1146  dateObs=dateObs)
1147  if self.config.doFlat else None)
1148 
1149  brighterFatterKernel = None
1150  brighterFatterGains = None
1151  if self.config.doBrighterFatter is True:
1152  try:
1153  # Use the new-style cp_pipe version of the kernel if it exists
1154  # If using a new-style kernel, always use the self-consistent
1155  # gains, i.e. the ones inside the kernel object itself
1156  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1157  brighterFatterGains = brighterFatterKernel.gain
1158  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1159  except NoResults:
1160  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1161  brighterFatterKernel = dataRef.get("bfKernel")
1162  self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1163  except NoResults:
1164  brighterFatterKernel = None
1165  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1166  # If the kernel is not an ndarray, it's the cp_pipe version
1167  # so extract the kernel for this detector, or raise an error
1168  if self.config.brighterFatterLevel == 'DETECTOR':
1169  if brighterFatterKernel.detKernels:
1170  brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1171  else:
1172  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1173  else:
1174  # TODO DM-15631 for implementing this
1175  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1176 
1177  defectList = (dataRef.get("defects")
1178  if self.config.doDefect else None)
1179  expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1180  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1181  if self.config.doAssembleIsrExposures else None)
1182  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1183  else pipeBase.Struct(fringes=None))
1184 
1185  if self.config.doAttachTransmissionCurve:
1186  opticsTransmission = (dataRef.get("transmission_optics")
1187  if self.config.doUseOpticsTransmission else None)
1188  filterTransmission = (dataRef.get("transmission_filter")
1189  if self.config.doUseFilterTransmission else None)
1190  sensorTransmission = (dataRef.get("transmission_sensor")
1191  if self.config.doUseSensorTransmission else None)
1192  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1193  if self.config.doUseAtmosphereTransmission else None)
1194  else:
1195  opticsTransmission = None
1196  filterTransmission = None
1197  sensorTransmission = None
1198  atmosphereTransmission = None
1199 
1200  if self.config.doStrayLight:
1201  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1202  else:
1203  strayLightData = None
1204 
1205  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1206  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1207  if (self.config.doIlluminationCorrection
1208  and physicalFilter in self.config.illumFilters)
1209  else None)
1210 
1211  # Struct should include only kwargs to run()
1212  return pipeBase.Struct(bias=biasExposure,
1213  linearizer=linearizer,
1214  crosstalk=crosstalkCalib,
1215  crosstalkSources=crosstalkSources,
1216  dark=darkExposure,
1217  flat=flatExposure,
1218  bfKernel=brighterFatterKernel,
1219  bfGains=brighterFatterGains,
1220  defects=defectList,
1221  fringes=fringeStruct,
1222  opticsTransmission=opticsTransmission,
1223  filterTransmission=filterTransmission,
1224  sensorTransmission=sensorTransmission,
1225  atmosphereTransmission=atmosphereTransmission,
1226  strayLightData=strayLightData,
1227  illumMaskedImage=illumMaskedImage
1228  )
1229 
1230  @pipeBase.timeMethod
1231  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1232  crosstalk=None, crosstalkSources=None,
1233  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1234  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1235  sensorTransmission=None, atmosphereTransmission=None,
1236  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1237  isGen3=False,
1238  ):
1239  """Perform instrument signature removal on an exposure.
1240 
1241  Steps included in the ISR processing, in order performed, are:
1242  - saturation and suspect pixel masking
1243  - overscan subtraction
1244  - CCD assembly of individual amplifiers
1245  - bias subtraction
1246  - variance image construction
1247  - linearization of non-linear response
1248  - crosstalk masking
1249  - brighter-fatter correction
1250  - dark subtraction
1251  - fringe correction
1252  - stray light subtraction
1253  - flat correction
1254  - masking of known defects and camera specific features
1255  - vignette calculation
1256  - appending transmission curve and distortion model
1257 
1258  Parameters
1259  ----------
1260  ccdExposure : `lsst.afw.image.Exposure`
1261  The raw exposure that is to be run through ISR. The
1262  exposure is modified by this method.
1263  camera : `lsst.afw.cameraGeom.Camera`, optional
1264  The camera geometry for this exposure. Required if ``isGen3`` is
1265  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1266  ``flat`` does not have an associated detector.
1267  bias : `lsst.afw.image.Exposure`, optional
1268  Bias calibration frame.
1269  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1270  Functor for linearization.
1271  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1272  Calibration for crosstalk.
1273  crosstalkSources : `list`, optional
1274  List of possible crosstalk sources.
1275  dark : `lsst.afw.image.Exposure`, optional
1276  Dark calibration frame.
1277  flat : `lsst.afw.image.Exposure`, optional
1278  Flat calibration frame.
1279  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1280  Photon transfer curve dataset, with, e.g., gains
1281  and read noise.
1282  bfKernel : `numpy.ndarray`, optional
1283  Brighter-fatter kernel.
1284  bfGains : `dict` of `float`, optional
1285  Gains used to override the detector's nominal gains for the
1286  brighter-fatter correction. A dict keyed by amplifier name for
1287  the detector in question.
1288  defects : `lsst.ip.isr.Defects`, optional
1289  List of defects.
1290  fringes : `lsst.pipe.base.Struct`, optional
1291  Struct containing the fringe correction data, with
1292  elements:
1293  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1294  - ``seed``: random seed derived from the ccdExposureId for random
1295  number generator (`uint32`)
1296  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1297  A ``TransmissionCurve`` that represents the throughput of the optics,
1298  to be evaluated in focal-plane coordinates.
1299  filterTransmission : `lsst.afw.image.TransmissionCurve`
1300  A ``TransmissionCurve`` that represents the throughput of the filter
1301  itself, to be evaluated in focal-plane coordinates.
1302  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1303  A ``TransmissionCurve`` that represents the throughput of the sensor
1304  itself, to be evaluated in post-assembly trimmed detector coordinates.
1305  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1306  A ``TransmissionCurve`` that represents the throughput of the
1307  atmosphere, assumed to be spatially constant.
1308  detectorNum : `int`, optional
1309  The integer number for the detector to process.
1310  isGen3 : bool, optional
1311  Flag this call to run() as using the Gen3 butler environment.
1312  strayLightData : `object`, optional
1313  Opaque object containing calibration information for stray-light
1314  correction. If `None`, no correction will be performed.
1315  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1316  Illumination correction image.
1317 
1318  Returns
1319  -------
1320  result : `lsst.pipe.base.Struct`
1321  Result struct with component:
1322  - ``exposure`` : `afw.image.Exposure`
1323  The fully ISR corrected exposure.
1324  - ``outputExposure`` : `afw.image.Exposure`
1325  An alias for `exposure`
1326  - ``ossThumb`` : `numpy.ndarray`
1327  Thumbnail image of the exposure after overscan subtraction.
1328  - ``flattenedThumb`` : `numpy.ndarray`
1329  Thumbnail image of the exposure after flat-field correction.
1330 
1331  Raises
1332  ------
1333  RuntimeError
1334  Raised if a configuration option is set to True, but the
1335  required calibration data has not been specified.
1336 
1337  Notes
1338  -----
1339  The current processed exposure can be viewed by setting the
1340  appropriate lsstDebug entries in the `debug.display`
1341  dictionary. The names of these entries correspond to some of
1342  the IsrTaskConfig Boolean options, with the value denoting the
1343  frame to use. The exposure is shown inside the matching
1344  option check and after the processing of that step has
1345  finished. The steps with debug points are:
1346 
1347  doAssembleCcd
1348  doBias
1349  doCrosstalk
1350  doBrighterFatter
1351  doDark
1352  doFringe
1353  doStrayLight
1354  doFlat
1355 
1356  In addition, setting the "postISRCCD" entry displays the
1357  exposure after all ISR processing has finished.
1358 
1359  """
1360 
1361  if isGen3 is True:
1362  # Gen3 currently cannot automatically do configuration overrides.
1363  # DM-15257 looks to discuss this issue.
1364  # Configure input exposures;
1365  if detectorNum is None:
1366  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1367 
1368  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1369  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1370  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1371  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1372  else:
1373  if isinstance(ccdExposure, ButlerDataRef):
1374  return self.runDataRefrunDataRef(ccdExposure)
1375 
1376  ccd = ccdExposure.getDetector()
1377  filterLabel = ccdExposure.getFilterLabel()
1378  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1379 
1380  if not ccd:
1381  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1382  ccd = [FakeAmp(ccdExposure, self.config)]
1383 
1384  # Validate Input
1385  if self.config.doBias and bias is None:
1386  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1387  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1388  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1389  if self.config.doBrighterFatter and bfKernel is None:
1390  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1391  if self.config.doDark and dark is None:
1392  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1393  if self.config.doFlat and flat is None:
1394  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1395  if self.config.doDefect and defects is None:
1396  raise RuntimeError("Must supply defects if config.doDefect=True.")
1397  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1398  and fringes.fringes is None):
1399  # The `fringes` object needs to be a pipeBase.Struct, as
1400  # we use it as a `dict` for the parameters of
1401  # `FringeTask.run()`. The `fringes.fringes` `list` may
1402  # not be `None` if `doFringe=True`. Otherwise, raise.
1403  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1404  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1405  and illumMaskedImage is None):
1406  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1407 
1408  # Begin ISR processing.
1409  if self.config.doConvertIntToFloat:
1410  self.log.info("Converting exposure to floating point values.")
1411  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1412 
1413  if self.config.doBias and self.config.doBiasBeforeOverscan:
1414  self.log.info("Applying bias correction.")
1415  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1416  trimToFit=self.config.doTrimToMatchCalib)
1417  self.debugViewdebugView(ccdExposure, "doBias")
1418 
1419  # Amplifier level processing.
1420  overscans = []
1421  for amp in ccd:
1422  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1423  if ccdExposure.getBBox().contains(amp.getBBox()):
1424  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1425  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1426 
1427  if self.config.doOverscan and not badAmp:
1428  # Overscan correction on amp-by-amp basis.
1429  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1430  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1431  if overscanResults is not None and \
1432  self.config.qa is not None and self.config.qa.saveStats is True:
1433  if isinstance(overscanResults.overscanFit, float):
1434  qaMedian = overscanResults.overscanFit
1435  qaStdev = float("NaN")
1436  else:
1437  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1438  afwMath.MEDIAN | afwMath.STDEVCLIP)
1439  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1440  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1441 
1442  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1443  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1444  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1445  amp.getName(), qaMedian, qaStdev)
1446 
1447  # Residuals after overscan correction
1448  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1449  afwMath.MEDIAN | afwMath.STDEVCLIP)
1450  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1451  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1452 
1453  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1454  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1455  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1456  amp.getName(), qaMedianAfter, qaStdevAfter)
1457 
1458  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1459  else:
1460  if badAmp:
1461  self.log.warning("Amplifier %s is bad.", amp.getName())
1462  overscanResults = None
1463 
1464  overscans.append(overscanResults if overscanResults is not None else None)
1465  else:
1466  self.log.info("Skipped OSCAN for %s.", amp.getName())
1467 
1468  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1469  self.log.info("Applying crosstalk correction.")
1470  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1471  crosstalkSources=crosstalkSources, camera=camera)
1472  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1473 
1474  if self.config.doAssembleCcd:
1475  self.log.info("Assembling CCD from amplifiers.")
1476  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1477 
1478  if self.config.expectWcs and not ccdExposure.getWcs():
1479  self.log.warning("No WCS found in input exposure.")
1480  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1481 
1482  ossThumb = None
1483  if self.config.qa.doThumbnailOss:
1484  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1485 
1486  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1487  self.log.info("Applying bias correction.")
1488  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1489  trimToFit=self.config.doTrimToMatchCalib)
1490  self.debugViewdebugView(ccdExposure, "doBias")
1491 
1492  if self.config.doVariance:
1493  for amp, overscanResults in zip(ccd, overscans):
1494  if ccdExposure.getBBox().contains(amp.getBBox()):
1495  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1496  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1497  if overscanResults is not None:
1498  self.updateVarianceupdateVariance(ampExposure, amp,
1499  overscanImage=overscanResults.overscanImage,
1500  ptcDataset=ptc)
1501  else:
1502  self.updateVarianceupdateVariance(ampExposure, amp,
1503  overscanImage=None,
1504  ptcDataset=ptc)
1505  if self.config.qa is not None and self.config.qa.saveStats is True:
1506  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1507  afwMath.MEDIAN | afwMath.STDEVCLIP)
1508  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1509  qaStats.getValue(afwMath.MEDIAN))
1510  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1511  qaStats.getValue(afwMath.STDEVCLIP))
1512  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1513  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1514  qaStats.getValue(afwMath.STDEVCLIP))
1515 
1516  if self.doLinearizedoLinearize(ccd):
1517  self.log.info("Applying linearizer.")
1518  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1519  detector=ccd, log=self.log)
1520 
1521  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1522  self.log.info("Applying crosstalk correction.")
1523  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1524  crosstalkSources=crosstalkSources, isTrimmed=True)
1525  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1526 
1527  # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1528  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1529  if self.config.doDefect:
1530  self.log.info("Masking defects.")
1531  self.maskDefectmaskDefect(ccdExposure, defects)
1532 
1533  if self.config.numEdgeSuspect > 0:
1534  self.log.info("Masking edges as SUSPECT.")
1535  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1536  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1537 
1538  if self.config.doNanMasking:
1539  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1540  self.maskNanmaskNan(ccdExposure)
1541 
1542  if self.config.doWidenSaturationTrails:
1543  self.log.info("Widening saturation trails.")
1544  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1545 
1546  if self.config.doCameraSpecificMasking:
1547  self.log.info("Masking regions for camera specific reasons.")
1548  self.masking.run(ccdExposure)
1549 
1550  if self.config.doBrighterFatter:
1551  # We need to apply flats and darks before we can interpolate, and we
1552  # need to interpolate before we do B-F, but we do B-F without the
1553  # flats and darks applied so we can work in units of electrons or holes.
1554  # This context manager applies and then removes the darks and flats.
1555  #
1556  # We also do not want to interpolate values here, so operate on temporary
1557  # images so we can apply only the BF-correction and roll back the
1558  # interpolation.
1559  interpExp = ccdExposure.clone()
1560  with self.flatContextflatContext(interpExp, flat, dark):
1561  isrFunctions.interpolateFromMask(
1562  maskedImage=interpExp.getMaskedImage(),
1563  fwhm=self.config.fwhm,
1564  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1565  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1566  )
1567  bfExp = interpExp.clone()
1568 
1569  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1570  type(bfKernel), type(bfGains))
1571  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1572  self.config.brighterFatterMaxIter,
1573  self.config.brighterFatterThreshold,
1574  self.config.brighterFatterApplyGain,
1575  bfGains)
1576  if bfResults[1] == self.config.brighterFatterMaxIter:
1577  self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1578  bfResults[0])
1579  else:
1580  self.log.info("Finished brighter-fatter correction in %d iterations.",
1581  bfResults[1])
1582  image = ccdExposure.getMaskedImage().getImage()
1583  bfCorr = bfExp.getMaskedImage().getImage()
1584  bfCorr -= interpExp.getMaskedImage().getImage()
1585  image += bfCorr
1586 
1587  # Applying the brighter-fatter correction applies a
1588  # convolution to the science image. At the edges this
1589  # convolution may not have sufficient valid pixels to
1590  # produce a valid correction. Mark pixels within the size
1591  # of the brighter-fatter kernel as EDGE to warn of this
1592  # fact.
1593  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1594  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1595  maskPlane="EDGE")
1596 
1597  if self.config.brighterFatterMaskGrowSize > 0:
1598  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1599  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1600  isrFunctions.growMasks(ccdExposure.getMask(),
1601  radius=self.config.brighterFatterMaskGrowSize,
1602  maskNameList=maskPlane,
1603  maskValue=maskPlane)
1604 
1605  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1606 
1607  if self.config.doDark:
1608  self.log.info("Applying dark correction.")
1609  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1610  self.debugViewdebugView(ccdExposure, "doDark")
1611 
1612  if self.config.doFringe and not self.config.fringeAfterFlat:
1613  self.log.info("Applying fringe correction before flat.")
1614  self.fringe.run(ccdExposure, **fringes.getDict())
1615  self.debugViewdebugView(ccdExposure, "doFringe")
1616 
1617  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1618  self.log.info("Checking strayLight correction.")
1619  self.strayLight.run(ccdExposure, strayLightData)
1620  self.debugViewdebugView(ccdExposure, "doStrayLight")
1621 
1622  if self.config.doFlat:
1623  self.log.info("Applying flat correction.")
1624  self.flatCorrectionflatCorrection(ccdExposure, flat)
1625  self.debugViewdebugView(ccdExposure, "doFlat")
1626 
1627  if self.config.doApplyGains:
1628  self.log.info("Applying gain correction instead of flat.")
1629  if self.config.usePtcGains:
1630  self.log.info("Using gains from the Photon Transfer Curve.")
1631  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1632  ptcGains=ptc.gain)
1633  else:
1634  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1635 
1636  if self.config.doFringe and self.config.fringeAfterFlat:
1637  self.log.info("Applying fringe correction after flat.")
1638  self.fringe.run(ccdExposure, **fringes.getDict())
1639 
1640  if self.config.doVignette:
1641  self.log.info("Constructing Vignette polygon.")
1642  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1643 
1644  if self.config.vignette.doWriteVignettePolygon:
1645  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1646 
1647  if self.config.doAttachTransmissionCurve:
1648  self.log.info("Adding transmission curves.")
1649  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1650  filterTransmission=filterTransmission,
1651  sensorTransmission=sensorTransmission,
1652  atmosphereTransmission=atmosphereTransmission)
1653 
1654  flattenedThumb = None
1655  if self.config.qa.doThumbnailFlattened:
1656  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1657 
1658  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1659  self.log.info("Performing illumination correction.")
1660  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1661  illumMaskedImage, illumScale=self.config.illumScale,
1662  trimToFit=self.config.doTrimToMatchCalib)
1663 
1664  preInterpExp = None
1665  if self.config.doSaveInterpPixels:
1666  preInterpExp = ccdExposure.clone()
1667 
1668  # Reset and interpolate bad pixels.
1669  #
1670  # Large contiguous bad regions (which should have the BAD mask
1671  # bit set) should have their values set to the image median.
1672  # This group should include defects and bad amplifiers. As the
1673  # area covered by these defects are large, there's little
1674  # reason to expect that interpolation would provide a more
1675  # useful value.
1676  #
1677  # Smaller defects can be safely interpolated after the larger
1678  # regions have had their pixel values reset. This ensures
1679  # that the remaining defects adjacent to bad amplifiers (as an
1680  # example) do not attempt to interpolate extreme values.
1681  if self.config.doSetBadRegions:
1682  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1683  if badPixelCount > 0:
1684  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1685 
1686  if self.config.doInterpolate:
1687  self.log.info("Interpolating masked pixels.")
1688  isrFunctions.interpolateFromMask(
1689  maskedImage=ccdExposure.getMaskedImage(),
1690  fwhm=self.config.fwhm,
1691  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1692  maskNameList=list(self.config.maskListToInterpolate)
1693  )
1694 
1695  self.roughZeroPointroughZeroPoint(ccdExposure)
1696 
1697  if self.config.doMeasureBackground:
1698  self.log.info("Measuring background level.")
1699  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1700 
1701  if self.config.qa is not None and self.config.qa.saveStats is True:
1702  for amp in ccd:
1703  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1704  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1705  afwMath.MEDIAN | afwMath.STDEVCLIP)
1706  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1707  qaStats.getValue(afwMath.MEDIAN))
1708  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1709  qaStats.getValue(afwMath.STDEVCLIP))
1710  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1711  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1712  qaStats.getValue(afwMath.STDEVCLIP))
1713 
1714  self.debugViewdebugView(ccdExposure, "postISRCCD")
1715 
1716  return pipeBase.Struct(
1717  exposure=ccdExposure,
1718  ossThumb=ossThumb,
1719  flattenedThumb=flattenedThumb,
1720 
1721  preInterpExposure=preInterpExp,
1722  outputExposure=ccdExposure,
1723  outputOssThumbnail=ossThumb,
1724  outputFlattenedThumbnail=flattenedThumb,
1725  )
1726 
1727  @pipeBase.timeMethod
1728  def runDataRef(self, sensorRef):
1729  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1730 
1731  This method contains the `CmdLineTask` interface to the ISR
1732  processing. All IO is handled here, freeing the `run()` method
1733  to manage only pixel-level calculations. The steps performed
1734  are:
1735  - Read in necessary detrending/isr/calibration data.
1736  - Process raw exposure in `run()`.
1737  - Persist the ISR-corrected exposure as "postISRCCD" if
1738  config.doWrite=True.
1739 
1740  Parameters
1741  ----------
1742  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1743  DataRef of the detector data to be processed
1744 
1745  Returns
1746  -------
1747  result : `lsst.pipe.base.Struct`
1748  Result struct with component:
1749  - ``exposure`` : `afw.image.Exposure`
1750  The fully ISR corrected exposure.
1751 
1752  Raises
1753  ------
1754  RuntimeError
1755  Raised if a configuration option is set to True, but the
1756  required calibration data does not exist.
1757 
1758  """
1759  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1760 
1761  ccdExposure = sensorRef.get(self.config.datasetType)
1762 
1763  camera = sensorRef.get("camera")
1764  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1765 
1766  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1767 
1768  if self.config.doWrite:
1769  sensorRef.put(result.exposure, "postISRCCD")
1770  if result.preInterpExposure is not None:
1771  sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1772  if result.ossThumb is not None:
1773  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1774  if result.flattenedThumb is not None:
1775  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1776 
1777  return result
1778 
1779  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1780  """Retrieve a calibration dataset for removing instrument signature.
1781 
1782  Parameters
1783  ----------
1784 
1785  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1786  DataRef of the detector data to find calibration datasets
1787  for.
1788  datasetType : `str`
1789  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1790  dateObs : `str`, optional
1791  Date of the observation. Used to correct butler failures
1792  when using fallback filters.
1793  immediate : `Bool`
1794  If True, disable butler proxies to enable error handling
1795  within this routine.
1796 
1797  Returns
1798  -------
1799  exposure : `lsst.afw.image.Exposure`
1800  Requested calibration frame.
1801 
1802  Raises
1803  ------
1804  RuntimeError
1805  Raised if no matching calibration frame can be found.
1806  """
1807  try:
1808  exp = dataRef.get(datasetType, immediate=immediate)
1809  except Exception as exc1:
1810  if not self.config.fallbackFilterName:
1811  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1812  try:
1813  if self.config.useFallbackDate and dateObs:
1814  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1815  dateObs=dateObs, immediate=immediate)
1816  else:
1817  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1818  except Exception as exc2:
1819  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1820  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1821  self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1822 
1823  if self.config.doAssembleIsrExposures:
1824  exp = self.assembleCcd.assembleCcd(exp)
1825  return exp
1826 
1827  def ensureExposure(self, inputExp, camera, detectorNum):
1828  """Ensure that the data returned by Butler is a fully constructed exposure.
1829 
1830  ISR requires exposure-level image data for historical reasons, so if we did
1831  not recieve that from Butler, construct it from what we have, modifying the
1832  input in place.
1833 
1834  Parameters
1835  ----------
1836  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1837  `lsst.afw.image.ImageF`
1838  The input data structure obtained from Butler.
1839  camera : `lsst.afw.cameraGeom.camera`
1840  The camera associated with the image. Used to find the appropriate
1841  detector.
1842  detectorNum : `int`
1843  The detector this exposure should match.
1844 
1845  Returns
1846  -------
1847  inputExp : `lsst.afw.image.Exposure`
1848  The re-constructed exposure, with appropriate detector parameters.
1849 
1850  Raises
1851  ------
1852  TypeError
1853  Raised if the input data cannot be used to construct an exposure.
1854  """
1855  if isinstance(inputExp, afwImage.DecoratedImageU):
1856  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1857  elif isinstance(inputExp, afwImage.ImageF):
1858  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1859  elif isinstance(inputExp, afwImage.MaskedImageF):
1860  inputExp = afwImage.makeExposure(inputExp)
1861  elif isinstance(inputExp, afwImage.Exposure):
1862  pass
1863  elif inputExp is None:
1864  # Assume this will be caught by the setup if it is a problem.
1865  return inputExp
1866  else:
1867  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1868  (type(inputExp), ))
1869 
1870  if inputExp.getDetector() is None:
1871  inputExp.setDetector(camera[detectorNum])
1872 
1873  return inputExp
1874 
1875  def convertIntToFloat(self, exposure):
1876  """Convert exposure image from uint16 to float.
1877 
1878  If the exposure does not need to be converted, the input is
1879  immediately returned. For exposures that are converted to use
1880  floating point pixels, the variance is set to unity and the
1881  mask to zero.
1882 
1883  Parameters
1884  ----------
1885  exposure : `lsst.afw.image.Exposure`
1886  The raw exposure to be converted.
1887 
1888  Returns
1889  -------
1890  newexposure : `lsst.afw.image.Exposure`
1891  The input ``exposure``, converted to floating point pixels.
1892 
1893  Raises
1894  ------
1895  RuntimeError
1896  Raised if the exposure type cannot be converted to float.
1897 
1898  """
1899  if isinstance(exposure, afwImage.ExposureF):
1900  # Nothing to be done
1901  self.log.debug("Exposure already of type float.")
1902  return exposure
1903  if not hasattr(exposure, "convertF"):
1904  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1905 
1906  newexposure = exposure.convertF()
1907  newexposure.variance[:] = 1
1908  newexposure.mask[:] = 0x0
1909 
1910  return newexposure
1911 
1912  def maskAmplifier(self, ccdExposure, amp, defects):
1913  """Identify bad amplifiers, saturated and suspect pixels.
1914 
1915  Parameters
1916  ----------
1917  ccdExposure : `lsst.afw.image.Exposure`
1918  Input exposure to be masked.
1919  amp : `lsst.afw.table.AmpInfoCatalog`
1920  Catalog of parameters defining the amplifier on this
1921  exposure to mask.
1922  defects : `lsst.ip.isr.Defects`
1923  List of defects. Used to determine if the entire
1924  amplifier is bad.
1925 
1926  Returns
1927  -------
1928  badAmp : `Bool`
1929  If this is true, the entire amplifier area is covered by
1930  defects and unusable.
1931 
1932  """
1933  maskedImage = ccdExposure.getMaskedImage()
1934 
1935  badAmp = False
1936 
1937  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1938  # comparison with current defects definition.
1939  if defects is not None:
1940  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1941 
1942  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1943  # association with pixels in current ccdExposure).
1944  if badAmp:
1945  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1946  afwImage.PARENT)
1947  maskView = dataView.getMask()
1948  maskView |= maskView.getPlaneBitMask("BAD")
1949  del maskView
1950  return badAmp
1951 
1952  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1953  # Saturation and suspect pixels can be masked now, though.
1954  limits = dict()
1955  if self.config.doSaturation and not badAmp:
1956  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1957  if self.config.doSuspect and not badAmp:
1958  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1959  if math.isfinite(self.config.saturation):
1960  limits.update({self.config.saturatedMaskName: self.config.saturation})
1961 
1962  for maskName, maskThreshold in limits.items():
1963  if not math.isnan(maskThreshold):
1964  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1965  isrFunctions.makeThresholdMask(
1966  maskedImage=dataView,
1967  threshold=maskThreshold,
1968  growFootprints=0,
1969  maskName=maskName
1970  )
1971 
1972  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1973  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1974  afwImage.PARENT)
1975  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1976  self.config.suspectMaskName])
1977  if numpy.all(maskView.getArray() & maskVal > 0):
1978  badAmp = True
1979  maskView |= maskView.getPlaneBitMask("BAD")
1980 
1981  return badAmp
1982 
1983  def overscanCorrection(self, ccdExposure, amp):
1984  """Apply overscan correction in place.
1985 
1986  This method does initial pixel rejection of the overscan
1987  region. The overscan can also be optionally segmented to
1988  allow for discontinuous overscan responses to be fit
1989  separately. The actual overscan subtraction is performed by
1990  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1991  which is called here after the amplifier is preprocessed.
1992 
1993  Parameters
1994  ----------
1995  ccdExposure : `lsst.afw.image.Exposure`
1996  Exposure to have overscan correction performed.
1997  amp : `lsst.afw.cameraGeom.Amplifer`
1998  The amplifier to consider while correcting the overscan.
1999 
2000  Returns
2001  -------
2002  overscanResults : `lsst.pipe.base.Struct`
2003  Result struct with components:
2004  - ``imageFit`` : scalar or `lsst.afw.image.Image`
2005  Value or fit subtracted from the amplifier image data.
2006  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2007  Value or fit subtracted from the overscan image data.
2008  - ``overscanImage`` : `lsst.afw.image.Image`
2009  Image of the overscan region with the overscan
2010  correction applied. This quantity is used to estimate
2011  the amplifier read noise empirically.
2012 
2013  Raises
2014  ------
2015  RuntimeError
2016  Raised if the ``amp`` does not contain raw pixel information.
2017 
2018  See Also
2019  --------
2020  lsst.ip.isr.isrFunctions.overscanCorrection
2021  """
2022  if amp.getRawHorizontalOverscanBBox().isEmpty():
2023  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2024  return None
2025 
2026  statControl = afwMath.StatisticsControl()
2027  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2028 
2029  # Determine the bounding boxes
2030  dataBBox = amp.getRawDataBBox()
2031  oscanBBox = amp.getRawHorizontalOverscanBBox()
2032  dx0 = 0
2033  dx1 = 0
2034 
2035  prescanBBox = amp.getRawPrescanBBox()
2036  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2037  dx0 += self.config.overscanNumLeadingColumnsToSkip
2038  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2039  else:
2040  dx0 += self.config.overscanNumTrailingColumnsToSkip
2041  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2042 
2043  # Determine if we need to work on subregions of the amplifier and overscan.
2044  imageBBoxes = []
2045  overscanBBoxes = []
2046 
2047  if ((self.config.overscanBiasJump
2048  and self.config.overscanBiasJumpLocation)
2049  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2050  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2051  self.config.overscanBiasJumpDevices)):
2052  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2053  yLower = self.config.overscanBiasJumpLocation
2054  yUpper = dataBBox.getHeight() - yLower
2055  else:
2056  yUpper = self.config.overscanBiasJumpLocation
2057  yLower = dataBBox.getHeight() - yUpper
2058 
2059  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2060  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2061  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2062  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2063  yLower)))
2064 
2065  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2066  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2067  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2068  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2069  yUpper)))
2070  else:
2071  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2072  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2073  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2074  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2075  oscanBBox.getHeight())))
2076 
2077  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2078  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2079  ampImage = ccdExposure.maskedImage[imageBBox]
2080  overscanImage = ccdExposure.maskedImage[overscanBBox]
2081 
2082  overscanArray = overscanImage.image.array
2083  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2084  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2085  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2086 
2087  statControl = afwMath.StatisticsControl()
2088  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2089 
2090  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2091 
2092  # Measure average overscan levels and record them in the metadata.
2093  levelStat = afwMath.MEDIAN
2094  sigmaStat = afwMath.STDEVCLIP
2095 
2096  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2097  self.config.qa.flatness.nIter)
2098  metadata = ccdExposure.getMetadata()
2099  ampNum = amp.getName()
2100  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2101  if isinstance(overscanResults.overscanFit, float):
2102  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2103  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2104  else:
2105  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2106  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2107  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2108 
2109  return overscanResults
2110 
2111  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2112  """Set the variance plane using the gain and read noise
2113 
2114  The read noise is calculated from the ``overscanImage`` if the
2115  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2116  the value from the amplifier data is used.
2117 
2118  Parameters
2119  ----------
2120  ampExposure : `lsst.afw.image.Exposure`
2121  Exposure to process.
2122  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2123  Amplifier detector data.
2124  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2125  Image of overscan, required only for empirical read noise.
2126  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2127  PTC dataset containing the gains and read noise.
2128 
2129 
2130  Raises
2131  ------
2132  RuntimeError
2133  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2134  are ``True``, but ptcDataset is not provided.
2135 
2136  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2137  ``overscanImage`` is ``None``.
2138 
2139  See also
2140  --------
2141  lsst.ip.isr.isrFunctions.updateVariance
2142  """
2143  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2144  if self.config.usePtcGains:
2145  if ptcDataset is None:
2146  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2147  else:
2148  gain = ptcDataset.gain[amp.getName()]
2149  self.log.info("Using gain from Photon Transfer Curve.")
2150  else:
2151  gain = amp.getGain()
2152 
2153  if math.isnan(gain):
2154  gain = 1.0
2155  self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2156  elif gain <= 0:
2157  patchedGain = 1.0
2158  self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2159  amp.getName(), gain, patchedGain)
2160  gain = patchedGain
2161 
2162  if self.config.doEmpiricalReadNoise and overscanImage is None:
2163  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2164 
2165  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2166  stats = afwMath.StatisticsControl()
2167  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2168  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2169  self.log.info("Calculated empirical read noise for amp %s: %f.",
2170  amp.getName(), readNoise)
2171  elif self.config.usePtcReadNoise:
2172  if ptcDataset is None:
2173  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2174  else:
2175  readNoise = ptcDataset.noise[amp.getName()]
2176  self.log.info("Using read noise from Photon Transfer Curve.")
2177  else:
2178  readNoise = amp.getReadNoise()
2179 
2180  isrFunctions.updateVariance(
2181  maskedImage=ampExposure.getMaskedImage(),
2182  gain=gain,
2183  readNoise=readNoise,
2184  )
2185 
2186  def darkCorrection(self, exposure, darkExposure, invert=False):
2187  """Apply dark correction in place.
2188 
2189  Parameters
2190  ----------
2191  exposure : `lsst.afw.image.Exposure`
2192  Exposure to process.
2193  darkExposure : `lsst.afw.image.Exposure`
2194  Dark exposure of the same size as ``exposure``.
2195  invert : `Bool`, optional
2196  If True, re-add the dark to an already corrected image.
2197 
2198  Raises
2199  ------
2200  RuntimeError
2201  Raised if either ``exposure`` or ``darkExposure`` do not
2202  have their dark time defined.
2203 
2204  See Also
2205  --------
2206  lsst.ip.isr.isrFunctions.darkCorrection
2207  """
2208  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2209  if math.isnan(expScale):
2210  raise RuntimeError("Exposure darktime is NAN.")
2211  if darkExposure.getInfo().getVisitInfo() is not None \
2212  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2213  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2214  else:
2215  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2216  # so getDarkTime() does not exist.
2217  self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2218  darkScale = 1.0
2219 
2220  isrFunctions.darkCorrection(
2221  maskedImage=exposure.getMaskedImage(),
2222  darkMaskedImage=darkExposure.getMaskedImage(),
2223  expScale=expScale,
2224  darkScale=darkScale,
2225  invert=invert,
2226  trimToFit=self.config.doTrimToMatchCalib
2227  )
2228 
2229  def doLinearize(self, detector):
2230  """Check if linearization is needed for the detector cameraGeom.
2231 
2232  Checks config.doLinearize and the linearity type of the first
2233  amplifier.
2234 
2235  Parameters
2236  ----------
2237  detector : `lsst.afw.cameraGeom.Detector`
2238  Detector to get linearity type from.
2239 
2240  Returns
2241  -------
2242  doLinearize : `Bool`
2243  If True, linearization should be performed.
2244  """
2245  return self.config.doLinearize and \
2246  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2247 
2248  def flatCorrection(self, exposure, flatExposure, invert=False):
2249  """Apply flat correction in place.
2250 
2251  Parameters
2252  ----------
2253  exposure : `lsst.afw.image.Exposure`
2254  Exposure to process.
2255  flatExposure : `lsst.afw.image.Exposure`
2256  Flat exposure of the same size as ``exposure``.
2257  invert : `Bool`, optional
2258  If True, unflatten an already flattened image.
2259 
2260  See Also
2261  --------
2262  lsst.ip.isr.isrFunctions.flatCorrection
2263  """
2264  isrFunctions.flatCorrection(
2265  maskedImage=exposure.getMaskedImage(),
2266  flatMaskedImage=flatExposure.getMaskedImage(),
2267  scalingType=self.config.flatScalingType,
2268  userScale=self.config.flatUserScale,
2269  invert=invert,
2270  trimToFit=self.config.doTrimToMatchCalib
2271  )
2272 
2273  def saturationDetection(self, exposure, amp):
2274  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2275 
2276  Parameters
2277  ----------
2278  exposure : `lsst.afw.image.Exposure`
2279  Exposure to process. Only the amplifier DataSec is processed.
2280  amp : `lsst.afw.table.AmpInfoCatalog`
2281  Amplifier detector data.
2282 
2283  See Also
2284  --------
2285  lsst.ip.isr.isrFunctions.makeThresholdMask
2286  """
2287  if not math.isnan(amp.getSaturation()):
2288  maskedImage = exposure.getMaskedImage()
2289  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2290  isrFunctions.makeThresholdMask(
2291  maskedImage=dataView,
2292  threshold=amp.getSaturation(),
2293  growFootprints=0,
2294  maskName=self.config.saturatedMaskName,
2295  )
2296 
2297  def saturationInterpolation(self, exposure):
2298  """Interpolate over saturated pixels, in place.
2299 
2300  This method should be called after `saturationDetection`, to
2301  ensure that the saturated pixels have been identified in the
2302  SAT mask. It should also be called after `assembleCcd`, since
2303  saturated regions may cross amplifier boundaries.
2304 
2305  Parameters
2306  ----------
2307  exposure : `lsst.afw.image.Exposure`
2308  Exposure to process.
2309 
2310  See Also
2311  --------
2312  lsst.ip.isr.isrTask.saturationDetection
2313  lsst.ip.isr.isrFunctions.interpolateFromMask
2314  """
2315  isrFunctions.interpolateFromMask(
2316  maskedImage=exposure.getMaskedImage(),
2317  fwhm=self.config.fwhm,
2318  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2319  maskNameList=list(self.config.saturatedMaskName),
2320  )
2321 
2322  def suspectDetection(self, exposure, amp):
2323  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2324 
2325  Parameters
2326  ----------
2327  exposure : `lsst.afw.image.Exposure`
2328  Exposure to process. Only the amplifier DataSec is processed.
2329  amp : `lsst.afw.table.AmpInfoCatalog`
2330  Amplifier detector data.
2331 
2332  See Also
2333  --------
2334  lsst.ip.isr.isrFunctions.makeThresholdMask
2335 
2336  Notes
2337  -----
2338  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2339  This is intended to indicate pixels that may be affected by unknown systematics;
2340  for example if non-linearity corrections above a certain level are unstable
2341  then that would be a useful value for suspectLevel. A value of `nan` indicates
2342  that no such level exists and no pixels are to be masked as suspicious.
2343  """
2344  suspectLevel = amp.getSuspectLevel()
2345  if math.isnan(suspectLevel):
2346  return
2347 
2348  maskedImage = exposure.getMaskedImage()
2349  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2350  isrFunctions.makeThresholdMask(
2351  maskedImage=dataView,
2352  threshold=suspectLevel,
2353  growFootprints=0,
2354  maskName=self.config.suspectMaskName,
2355  )
2356 
2357  def maskDefect(self, exposure, defectBaseList):
2358  """Mask defects using mask plane "BAD", in place.
2359 
2360  Parameters
2361  ----------
2362  exposure : `lsst.afw.image.Exposure`
2363  Exposure to process.
2364  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2365  `lsst.afw.image.DefectBase`.
2366  List of defects to mask.
2367 
2368  Notes
2369  -----
2370  Call this after CCD assembly, since defects may cross amplifier boundaries.
2371  """
2372  maskedImage = exposure.getMaskedImage()
2373  if not isinstance(defectBaseList, Defects):
2374  # Promotes DefectBase to Defect
2375  defectList = Defects(defectBaseList)
2376  else:
2377  defectList = defectBaseList
2378  defectList.maskPixels(maskedImage, maskName="BAD")
2379 
2380  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2381  """Mask edge pixels with applicable mask plane.
2382 
2383  Parameters
2384  ----------
2385  exposure : `lsst.afw.image.Exposure`
2386  Exposure to process.
2387  numEdgePixels : `int`, optional
2388  Number of edge pixels to mask.
2389  maskPlane : `str`, optional
2390  Mask plane name to use.
2391  level : `str`, optional
2392  Level at which to mask edges.
2393  """
2394  maskedImage = exposure.getMaskedImage()
2395  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2396 
2397  if numEdgePixels > 0:
2398  if level == 'DETECTOR':
2399  boxes = [maskedImage.getBBox()]
2400  elif level == 'AMP':
2401  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2402 
2403  for box in boxes:
2404  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2405  subImage = maskedImage[box]
2406  box.grow(-numEdgePixels)
2407  # Mask pixels outside box
2408  SourceDetectionTask.setEdgeBits(
2409  subImage,
2410  box,
2411  maskBitMask)
2412 
2413  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2414  """Mask and interpolate defects using mask plane "BAD", in place.
2415 
2416  Parameters
2417  ----------
2418  exposure : `lsst.afw.image.Exposure`
2419  Exposure to process.
2420  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2421  `lsst.afw.image.DefectBase`.
2422  List of defects to mask and interpolate.
2423 
2424  See Also
2425  --------
2426  lsst.ip.isr.isrTask.maskDefect
2427  """
2428  self.maskDefectmaskDefect(exposure, defectBaseList)
2429  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2430  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2431  isrFunctions.interpolateFromMask(
2432  maskedImage=exposure.getMaskedImage(),
2433  fwhm=self.config.fwhm,
2434  growSaturatedFootprints=0,
2435  maskNameList=["BAD"],
2436  )
2437 
2438  def maskNan(self, exposure):
2439  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2440 
2441  Parameters
2442  ----------
2443  exposure : `lsst.afw.image.Exposure`
2444  Exposure to process.
2445 
2446  Notes
2447  -----
2448  We mask over all non-finite values (NaN, inf), including those
2449  that are masked with other bits (because those may or may not be
2450  interpolated over later, and we want to remove all NaN/infs).
2451  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2452  preserve the historical name.
2453  """
2454  maskedImage = exposure.getMaskedImage()
2455 
2456  # Find and mask NaNs
2457  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2458  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2459  numNans = maskNans(maskedImage, maskVal)
2460  self.metadata.set("NUMNANS", numNans)
2461  if numNans > 0:
2462  self.log.warning("There were %d unmasked NaNs.", numNans)
2463 
2464  def maskAndInterpolateNan(self, exposure):
2465  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2466  in place.
2467 
2468  Parameters
2469  ----------
2470  exposure : `lsst.afw.image.Exposure`
2471  Exposure to process.
2472 
2473  See Also
2474  --------
2475  lsst.ip.isr.isrTask.maskNan
2476  """
2477  self.maskNanmaskNan(exposure)
2478  isrFunctions.interpolateFromMask(
2479  maskedImage=exposure.getMaskedImage(),
2480  fwhm=self.config.fwhm,
2481  growSaturatedFootprints=0,
2482  maskNameList=["UNMASKEDNAN"],
2483  )
2484 
2485  def measureBackground(self, exposure, IsrQaConfig=None):
2486  """Measure the image background in subgrids, for quality control purposes.
2487 
2488  Parameters
2489  ----------
2490  exposure : `lsst.afw.image.Exposure`
2491  Exposure to process.
2492  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2493  Configuration object containing parameters on which background
2494  statistics and subgrids to use.
2495  """
2496  if IsrQaConfig is not None:
2497  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2498  IsrQaConfig.flatness.nIter)
2499  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2500  statsControl.setAndMask(maskVal)
2501  maskedImage = exposure.getMaskedImage()
2502  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2503  skyLevel = stats.getValue(afwMath.MEDIAN)
2504  skySigma = stats.getValue(afwMath.STDEVCLIP)
2505  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2506  metadata = exposure.getMetadata()
2507  metadata.set('SKYLEVEL', skyLevel)
2508  metadata.set('SKYSIGMA', skySigma)
2509 
2510  # calcluating flatlevel over the subgrids
2511  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2512  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2513  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2514  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2515  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2516  skyLevels = numpy.zeros((nX, nY))
2517 
2518  for j in range(nY):
2519  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2520  for i in range(nX):
2521  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2522 
2523  xLLC = xc - meshXHalf
2524  yLLC = yc - meshYHalf
2525  xURC = xc + meshXHalf - 1
2526  yURC = yc + meshYHalf - 1
2527 
2528  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2529  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2530 
2531  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2532 
2533  good = numpy.where(numpy.isfinite(skyLevels))
2534  skyMedian = numpy.median(skyLevels[good])
2535  flatness = (skyLevels[good] - skyMedian) / skyMedian
2536  flatness_rms = numpy.std(flatness)
2537  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2538 
2539  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2540  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2541  nX, nY, flatness_pp, flatness_rms)
2542 
2543  metadata.set('FLATNESS_PP', float(flatness_pp))
2544  metadata.set('FLATNESS_RMS', float(flatness_rms))
2545  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2546  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2547  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2548 
2549  def roughZeroPoint(self, exposure):
2550  """Set an approximate magnitude zero point for the exposure.
2551 
2552  Parameters
2553  ----------
2554  exposure : `lsst.afw.image.Exposure`
2555  Exposure to process.
2556  """
2557  filterLabel = exposure.getFilterLabel()
2558  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2559 
2560  if physicalFilter in self.config.fluxMag0T1:
2561  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2562  else:
2563  self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2564  fluxMag0 = self.config.defaultFluxMag0T1
2565 
2566  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2567  if not expTime > 0: # handle NaN as well as <= 0
2568  self.log.warning("Non-positive exposure time; skipping rough zero point.")
2569  return
2570 
2571  self.log.info("Setting rough magnitude zero point for filter %s: %f",
2572  physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2573  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2574 
2575  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2576  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2577 
2578  Parameters
2579  ----------
2580  ccdExposure : `lsst.afw.image.Exposure`
2581  Exposure to process.
2582  fpPolygon : `lsst.afw.geom.Polygon`
2583  Polygon in focal plane coordinates.
2584  """
2585  # Get ccd corners in focal plane coordinates
2586  ccd = ccdExposure.getDetector()
2587  fpCorners = ccd.getCorners(FOCAL_PLANE)
2588  ccdPolygon = Polygon(fpCorners)
2589 
2590  # Get intersection of ccd corners with fpPolygon
2591  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2592 
2593  # Transform back to pixel positions and build new polygon
2594  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2595  validPolygon = Polygon(ccdPoints)
2596  ccdExposure.getInfo().setValidPolygon(validPolygon)
2597 
2598  @contextmanager
2599  def flatContext(self, exp, flat, dark=None):
2600  """Context manager that applies and removes flats and darks,
2601  if the task is configured to apply them.
2602 
2603  Parameters
2604  ----------
2605  exp : `lsst.afw.image.Exposure`
2606  Exposure to process.
2607  flat : `lsst.afw.image.Exposure`
2608  Flat exposure the same size as ``exp``.
2609  dark : `lsst.afw.image.Exposure`, optional
2610  Dark exposure the same size as ``exp``.
2611 
2612  Yields
2613  ------
2614  exp : `lsst.afw.image.Exposure`
2615  The flat and dark corrected exposure.
2616  """
2617  if self.config.doDark and dark is not None:
2618  self.darkCorrectiondarkCorrection(exp, dark)
2619  if self.config.doFlat:
2620  self.flatCorrectionflatCorrection(exp, flat)
2621  try:
2622  yield exp
2623  finally:
2624  if self.config.doFlat:
2625  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2626  if self.config.doDark and dark is not None:
2627  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2628 
2629  def debugView(self, exposure, stepname):
2630  """Utility function to examine ISR exposure at different stages.
2631 
2632  Parameters
2633  ----------
2634  exposure : `lsst.afw.image.Exposure`
2635  Exposure to view.
2636  stepname : `str`
2637  State of processing to view.
2638  """
2639  frame = getDebugFrame(self._display, stepname)
2640  if frame:
2641  display = getDisplay(frame)
2642  display.scale('asinh', 'zscale')
2643  display.mtv(exposure)
2644  prompt = "Press Enter to continue [c]... "
2645  while True:
2646  ans = input(prompt).lower()
2647  if ans in ("", "c",):
2648  break
2649 
2650 
2651 class FakeAmp(object):
2652  """A Detector-like object that supports returning gain and saturation level
2653 
2654  This is used when the input exposure does not have a detector.
2655 
2656  Parameters
2657  ----------
2658  exposure : `lsst.afw.image.Exposure`
2659  Exposure to generate a fake amplifier for.
2660  config : `lsst.ip.isr.isrTaskConfig`
2661  Configuration to apply to the fake amplifier.
2662  """
2663 
2664  def __init__(self, exposure, config):
2665  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2666  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2667  self._gain_gain = config.gain
2668  self._readNoise_readNoise = config.readNoise
2669  self._saturation_saturation = config.saturation
2670 
2671  def getBBox(self):
2672  return self._bbox_bbox
2673 
2674  def getRawBBox(self):
2675  return self._bbox_bbox
2676 
2678  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2679 
2680  def getGain(self):
2681  return self._gain_gain
2682 
2683  def getReadNoise(self):
2684  return self._readNoise_readNoise
2685 
2686  def getSaturation(self):
2687  return self._saturation_saturation
2688 
2689  def getSuspectLevel(self):
2690  return float("NaN")
2691 
2692 
2693 class RunIsrConfig(pexConfig.Config):
2694  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2695 
2696 
2697 class RunIsrTask(pipeBase.CmdLineTask):
2698  """Task to wrap the default IsrTask to allow it to be retargeted.
2699 
2700  The standard IsrTask can be called directly from a command line
2701  program, but doing so removes the ability of the task to be
2702  retargeted. As most cameras override some set of the IsrTask
2703  methods, this would remove those data-specific methods in the
2704  output post-ISR images. This wrapping class fixes the issue,
2705  allowing identical post-ISR images to be generated by both the
2706  processCcd and isrTask code.
2707  """
2708  ConfigClass = RunIsrConfig
2709  _DefaultName = "runIsr"
2710 
2711  def __init__(self, *args, **kwargs):
2712  super().__init__(*args, **kwargs)
2713  self.makeSubtask("isr")
2714 
2715  def runDataRef(self, dataRef):
2716  """
2717  Parameters
2718  ----------
2719  dataRef : `lsst.daf.persistence.ButlerDataRef`
2720  data reference of the detector data to be processed
2721 
2722  Returns
2723  -------
2724  result : `pipeBase.Struct`
2725  Result struct with component:
2726 
2727  - exposure : `lsst.afw.image.Exposure`
2728  Post-ISR processed exposure.
2729  """
2730  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2677
def __init__(self, exposure, config)
Definition: isrTask.py:2664
def __init__(self, *config=None)
Definition: isrTask.py:272
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2248
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2464
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2297
def runDataRef(self, sensorRef)
Definition: isrTask.py:1728
def maskNan(self, exposure)
Definition: isrTask.py:2438
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1912
def debugView(self, exposure, stepname)
Definition: isrTask.py:2629
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1779
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2273
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2357
def __init__(self, **kwargs)
Definition: isrTask.py:948
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:958
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2380
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1983
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2485
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2549
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2413
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2575
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1056
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1827
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1238
def doLinearize(self, detector)
Definition: isrTask.py:2229
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2599
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1875
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2322
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2111
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2186
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2711
def runDataRef(self, dataRef)
Definition: isrTask.py:2715
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35